<file_sep># -*- coding: utf-8 -*-
"""
Created on Thu Mar  4 15:11:35 2021

@author: Hania
"""
import os
import time
import logging 
import sys
import shutil
import numpy as np
import json
import pickle

from sklearn.calibration import CalibratedClassifierCV

def save_configs(cfgs_list, directory):
    # stores config files in the experiment dir
    timestamp = time.strftime('%Y-%m-%d-%H-%M')
    for config_file in cfgs_list:
        filename = f"{timestamp}-{os.path.basename(config_file)}"
        shutil.copyfile(config_file, os.path.join(directory, filename))


def save_as_json(obj, saving_dir, filename, nexp=None):
    # saves with json but uses a timestamp
    ## nexp = optional neptune experiment
    timestamp = time.strftime('%Y-%m-%d-%H-%M')
    if isinstance(obj, dict):
        for key in obj.keys():
            if isinstance(obj[key], np.ndarray):
                obj[key] = obj[key].tolist()

    with open(os.path.join(saving_dir, f'{timestamp}-{filename}'), 'w') as f:
        json.dump(obj, f, indent=2)
        
    if nexp:
        nexp.log_artifact(os.path.join(saving_dir, f'{timestamp}-{filename}'))


def pickle_and_log_artifact(obj, saving_dir, filename, nexp=None):
    timestamp = time.strftime('%Y-%m-%d-%H-%M')
    with open(os.path.join(saving_dir, f'{timestamp}-{filename}.pickle'), 'wb') as f:
        pickle.dump(obj, f)
    if nexp is not None:
        nexp.log_artifact(os.path.join(saving_dir, f'{timestamp}-{filename}.pickle'))


def save_predictions(x, y, cv_split, test_x, test_y, model, saving_dir):
    timestamp = time.strftime('%Y-%m-%d-%H-%M')

    def _save_to_file(true_label, predicted_probabilities, filename):
        # if predicted_probabilities is None:
        #    predicted_probabilities = [None, ] * len(smiles)
        try:
            os.makedirs(saving_dir)
        except FileExistsError:
            pass
        with open(os.path.join(saving_dir, f"{timestamp}-{filename}"), 'w') as fid:
            fid.write('has_bird\tbird_probability\n')
           # print((predicted_probabilities))
           # print(np.shape(predicted_probabilities))
            if np.shape(predicted_probabilities)[1]==1:
                pp = predicted_probabilities[:,0];
            else:
                pp = predicted_probabilities[:,1];
            for true, proba in zip(true_label, pp): #predicted_probabilities[:,1]
                fid.write(f"{true}\t{proba}\n")
               
    # test data
    # predicted = model.predict(test_x)
    try:
        proba = model.predict_proba(test_x)
    except (AttributeError, RuntimeError):
        #proba = None
        clf = CalibratedClassifierCV(model.fitted_pipeline_)  # !!! zmiana dla LinearSVC. to samo jak dla SVC probability=True
        clf.fit(x,y) 
        proba = clf.predict_proba(test_x)
        
    _save_to_file(test_y, proba, f'test.predictions')
    
        # training data
    for idx, (_, indices) in enumerate(cv_split):
        this_x = x[indices]
        this_y = y[indices]
        # this_smiles = smiles[indices]
        # predicted = model.predict(this_x) # klasy mnie nie interesują
        try:
            proba = model.predict_proba(this_x)
        except (AttributeError, RuntimeError):
            #proba = None   
            proba = clf.predict_proba(this_x) 
            
        _save_to_file(this_y, proba, f'train-valid-{idx}.predictions')

# moje
def my_save_to_file(true_label, predicted_probabilities, filename, saving_dir):
    #timestamp = saving_dir[-16:] #time.strftime('%Y-%m-%d-%H-%M')
    #print(timestamp)
    try:
        os.makedirs(saving_dir)
    except FileExistsError:
        pass
    with open(os.path.join(saving_dir, f"{filename}"), 'w') as fid: #{timestamp}-
        fid.write('has_bird\tbird_probability\n')
       # print((predicted_probabilities))
       # print(np.shape(predicted_probabilities))
        for true, proba in zip(true_label, predicted_probabilities):
            fid.write(f"{true}\t{proba}\n")


class LoggerWrapper:
    def __init__(self, path='.'):
        """
        Wrapper for logging.
        Allows to replace sys.stderr.write so that error massages are redirected do sys.stdout and also saved in a file.
        use: logger = LoggerWrapper(); sys.stderr.write = logger.log_errors
        :param: path: directory to create log file
        """
        # count spaces so that the output is nicely indented
        self.trailing_spaces = 0

        # create the log file
        timestamp = time.strftime('%Y-%m-%d-%H-%M')
        self.filename = os.path.join(path, f'{timestamp}.log')
        try:
            # os.mknod(self.filename)
            with open(self.filename,"w") as f:
                pass
        except FileExistsError:
            pass

        # configure logging
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s %(name)-6s %(levelname)-8s %(message)s',
                            datefmt='%m-%d %H:%M',
                            filename=self.filename,
                            filemode='w')
        formatter = logging.Formatter('%(name)-6s: %(levelname)-8s %(message)s')

        # make a handler to redirect stuff to std.out
        self.logger = logging.getLogger('')
        self.logger.setLevel(logging.INFO)  # bacause matplotlib throws lots of debug messages
        self.console = logging.StreamHandler(sys.stdout)
        self.console.setLevel(logging.INFO)
        self.console.setFormatter(formatter)
        self.logger.addHandler(self.console)

    def log_errors(self, msg):
        msg = msg.strip('\n')  # don't add extra enters

        if msg == ' ' * len(msg):  # if you only get spaces: don't print them, but do remember
            self.trailing_spaces += len(msg)
        elif len(msg) > 1:
            self.logger.error(' ' * self.trailing_spaces + msg)
            self.trailing_spaces = 0
            
            
<file_sep># -*- coding: utf-8 -*-
"""
Created on Tue Mar 23 13:29:58 2021

@author: Hania
"""

# IMPORTY
import joblib
import os

import numpy as np
import librosa
import time

def funkcja_reprezentacja(path_wav, data_settype, representation_type, if_scaler, repr_1d_summary, summary_1d, sr, chunk_length_ms, \
                          n_fft, win_length, hop_length, window, f_min, f_max, n_mels, N, step, Np, K, tm, flock, tlock, nr_nagr_pocz, wielkosc_batcha):
   
  '''
  Jest to funkcja zawierająca cały potrzebny kod z pliku 'reprezentacja śmieci/Untitled1', bez zbędnych printów, komentarzy itd.

  Args: 
    te wszystkie parametry wejściowe co powyżej, opisane w miarę w mainie

  Returns:
    file_names_train_set - nazwy nagrań wchodzące w skład zbioru
    indices - indeksy chunksów wybrane z poszczególnych nagrań
    info_chunksy - informacje o wybranych chunksach - te dane co wcześniej w dataframe, ale tylko dla wybranych chunksów 
      ['chunk_ids', 'chunk_start', 'chunk_end', 'has_bird', 'chunks_species', 'call_id', 'has_unknown', 'has_noise']
    representation_type - - to co na wejściu było, jaki jeden z pięciu typów reprezentacji
    repr_full  - wybrana reprezentacja dla każdego z wybranych chunksów z nagrań. Rozmiary:
      spektrogram: 1 nagranie - chunksy x 63 x 148
      mel-spektrogram: 1 nagranie - chunksy x 60 x 148
      multitaper: 1 nagranie - chunksy x 64 x 149
      8_classic: 1 nagranie - chunksy x 1-4 x 8
      8_classic_plus_MIR: 1 nagranie - chunksy x 1-4 x 39

  ''' 

  ## FUNKCJE
 
  def my_spektrogram(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, scaler = None):
    stft = librosa.stft(y, n_fft=n_fft, win_length=win_length, hop_length=hop_length, window= window) # krótkoczasowa transformata fouriera STFT
    stft1 = librosa.amplitude_to_db(np.abs(stft)**2)                                              # kwadrat wartości bezwzględnej plus przerzucenie na skalę decybelową
    freqs = librosa.core.fft_frequencies(n_fft=n_fft, sr=sr)                             # wyznaczenie częstotliwości
    x,  = np.where( freqs >= min(freqs[(freqs >= f_min)]))
    j,  = np.where( freqs <= max(freqs[(freqs <= f_max)]))
    stft1 = stft1[min(x):max(j),]                                                        # stft tylko w wybranym zakresie
    #repr1_spectro = stft1
    if np.shape(stft1)[1]!= 148:
      stft1 = np.pad(stft1, ((0, 0), (0, 148 - np.shape(stft1)[1])), 'constant', constant_values=(-100))
      print("padding do ",np.shape(stft1))
    if (scaler!=None):
      return scaler.transform(stft1)   #repr1_spectro
    else:
      return stft1
 
 
  def my_melspektrogram(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, n_mels, scaler = None):
    stft = librosa.stft(np.array(y), n_fft=n_fft, win_length=win_length, hop_length= hop_length, window=window)
    abs2_stft = (stft.real*stft.real) + (stft.imag*stft.imag)
    melspect = librosa.feature.melspectrogram(y=None, S=abs2_stft, sr=sr, n_mels= n_mels, fmin = f_min, fmax=f_max, hop_length=hop_length, n_fft=n_fft)
    repr2_melspec = 0.5 * librosa.amplitude_to_db(melspect, ref=1.0)
    if np.shape(repr2_melspec)[1]!= 148:
      repr2_melspec = np.pad(repr2_melspec, ((0, 0), (0, 148 - np.shape(repr2_melspec)[1])), 'constant', constant_values=(-50))    
    if (scaler!=None):
      return scaler.transform(repr2_melspec)   #repr1_spectro
    else:
      return repr2_melspec
  '''
  def my_multitaper(y, N, step, Np, K, tm, flock, tlock, f_min, f_max, scaler = None): 
    result5b = libtfr.tfr_spec(y, N = N, step = step, Np = Np, K = 2, tm = tm, flock = flock, tlock = tlock)     
    freqs, ind = libtfr.fgrid(sr, N, fpass=(f_min,f_max)) 
    repr3_multitaper = librosa.amplitude_to_db(result5b[ind,]); # tylko interesujące nas pasmo, w log
    #stft1 = librosa.amplitude_to_db(np.abs(stft)**2) 
    if np.shape(repr3_multitaper)[1]!= 149:
      repr3_multitaper = np.pad(repr3_multitaper, ((0, 0), (0, 149 - np.shape(repr3_multitaper)[1])), 'constant', constant_values=(-100))    
    if (scaler!=None):
      return scaler.transform(repr3_multitaper )   #repr1_spectro
    else:
      return repr3_multitaper
  '''
  def FeatureSpectralFlux(X):  #https://www.audiocontentanalysis.org/code/audio-features/spectral-flux-2/
    # difference spectrum (set first diff to zero)
    X = np.c_[X[:, 0], X]
    afDeltaX = np.diff(X, 1, axis=1)
    # flux
    vsf = np.sqrt((afDeltaX**2).sum(axis=0)) / X.shape[0]
    return  vsf[1:]                 # pozbycie się pierwszego elementu, który zawsze jest zerem , np.squeeze(vsf[1:]) if isSpectrum else

  def classic_base(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, scaler4 = None):
    S1 = np.abs(librosa.stft(np.array(y), n_fft=n_fft, win_length=win_length, hop_length= hop_length, window=window))
    freqs = librosa.core.fft_frequencies(n_fft=n_fft, sr=sr) 
    o,  = np.where( freqs >= min(freqs[(freqs >= f_min)]))
    j,  = np.where( freqs <= max(freqs[(freqs <= f_max)]))
    freqs1 = freqs[min(o):max(j),]
    S = S1[min(o):max(j),] 

    param_0 = np.sum(S, axis=0)             #librosa.feature.spectral_bandwidth(S=S, p = 1)  # moc sygnału. Ale z wartości absolutnej spektorgramu, bez tego kwadratu jeszcz
    param_1 = librosa.feature.spectral_centroid(S=S, freq=freqs1)                            # centroid https://www.mathworks.com/help/audio/ref/spectralcentroid.html
    param_2 = np.power(librosa.feature.spectral_bandwidth(S=S, freq=freqs1, p = 2),2)        # 2 rzędu  
    param_3 = np.power(librosa.feature.spectral_bandwidth(S=S, freq=freqs1, p = 3),3)        # 3 rzędu
    param_4 = np.power(librosa.feature.spectral_bandwidth(S=S, freq=freqs1, p = 4),4)        # 4 rzędu
    skosnosc = param_3[0] / np.power(param_2[0],1.5)                        # https://www.mathworks.com/help/audio/ref/spectralskewness.html #skosnosc2 = skew(S, axis=0)
    kurtoza =  param_4[0]/ np.power(param_2[0],2) - 3                       #kurtoza2 = kurtosis(S, axis=0)
    plaskosc = librosa.feature.spectral_flatness(S=S)                       #gmean(S_squared)/np.mean(S_squared)

    return param_0, param_1, param_2, param_3, param_4, skosnosc, kurtoza, plaskosc, S


  def my_8_classic(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, summary_1d, scaler4 = None): ## TO DO scalers

    param_0, param_1, param_2, param_3, param_4, skosnosc, kurtoza, plaskosc, _ = classic_base(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, scaler4)
    nb_summary = np.sum(summary_1d)
    paramsy = [[[] for _ in range(8)] for _ in range(nb_summary)]
    idx = 0

    for m in range(np.shape(summary_1d)[0]):
      if summary_1d[m]:
        f = getattr(np, repr_1d_summary[m])
        paramsy[idx]=[f(param_0), f(param_1), f(param_2), f(param_3), f(param_4), f(skosnosc), f(kurtoza), f(plaskosc)]
        idx += 1
        
    if (scaler4!=None):
        return scaler4.transform(paramsy)
    else:
        return paramsy

  def my_8_classic_plus_MIR(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, summary_1d, scaler5 = None): ## TO DO scalers
    param_0, param_1, param_2, param_3, param_4, skosnosc, kurtoza, plaskosc, S = classic_base(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, scaler4)
    nb_summary = np.sum(summary_1d)
    stft = librosa.stft(np.array(y), n_fft=n_fft, win_length=win_length, hop_length= hop_length, window=window)
    abs2_stft = (stft.real*stft.real) + (stft.imag*stft.imag)
    melspect = librosa.feature.melspectrogram(y=None, S=abs2_stft, sr=sr, n_mels= n_mels, fmin = f_min, fmax=f_max, hop_length=hop_length, n_fft=n_fft)
    
    mfccs =librosa.feature.mfcc(S=librosa.power_to_db(melspect), n_mfcc=12)
    mfcc_delta = librosa.feature.delta(mfccs)
    zcr = sum(librosa.feature.zero_crossing_rate(y, frame_length=win_length, hop_length= hop_length))  #  ZCR can be interpreted as a measure of the noisiness of a signal. For example, it usually exhibits higher values in the case of noisy signals. It is also known to reflect, in a rather coarse manner, the spectral characteristics of a signal. std to recognize speech vs music
    contrast = librosa.feature.spectral_contrast(S=S, hop_length=hop_length, n_fft=n_fft, n_bands = 2, fmin = f_min)
    rolloff = librosa.feature.spectral_rolloff(S=S, hop_length=hop_length, n_fft=n_fft, roll_percent=0.85)    
    rms = librosa.feature.rms(S=S, hop_length=hop_length, frame_length = 124) 
    spectral_flux = FeatureSpectralFlux(S)
    np.shape(rms)
    paramsy = [[[] for _ in range(39)] for _ in range(nb_summary)]
    idx = 0

    for m in range(np.shape(summary_1d)[0]):
      if summary_1d[m]:
        f = getattr(np, repr_1d_summary[m])  # która statystyka wybrana repr_1d_summary = ['min', 'max', 'mean', 'std']
        paramsy_mir = [f(param_0), f(param_1), f(param_2), f(param_3), f(param_4), f(skosnosc), f(kurtoza), f(plaskosc)]
        paramsy_mir.extend(f(mfccs, axis = 1).tolist())
        paramsy_mir.extend(f(mfcc_delta, axis = 1).tolist())
        paramsy_mir.extend([f(zcr)])
        paramsy_mir.extend(f(contrast, axis = 1).tolist())
        paramsy_mir.extend([f(rolloff), f(rms), f(spectral_flux)])
        paramsy[idx]= paramsy_mir
        idx += 1

    if (scaler5!=None):
        return scaler5.transform(paramsy)
    else:
        return paramsy


  # nie powinno mieć to wszystko train w nazwie, no ale już trudno. Chodzi o dowolny dataset który będzie na wejściu 
  # z funkcji wczytanie:
  file_names_train_set = data_settype[0]
  indices = data_settype[1]
  result_dataframe_train = data_settype[2]  # trzeba powyciągać tylko interesujące nas chunksy
  df_to_np = result_dataframe_train.to_numpy()  # bo nie umiem sobie poradiś jak jest to df, wiele funkcji które chcę użyć nie działa


  repr_full = [[] for _ in range(np.shape(file_names_train_set)[0])]
  #repr_full1, repr_full2, repr_full3 = [[] for _ in range(np.shape(file_names_train_set)[0])],[[] for _ in range(np.shape(file_names_train_set)[0])],[[] for _ in range(np.shape(file_names_train_set)[0])]
  #repr_full4, repr_full5 = [[] for _ in range(np.shape(file_names_train_set)[0])], [[] for _ in range(np.shape(file_names_train_set)[0])]
  info_chunksy = [[[] for _ in range(8)] for _ in range(np.shape(file_names_train_set)[0])]  # 8 kolumn w dataframie było
 
  print(file_names_train_set) 

  # by nie wczytywać scalerów przy każdym chunku albo nagraniu. True - mamy już wyznaczone scalery,  i będziemy wszystko normalizować scalerami z traina. Jeśli false, to bez normalizacji:
  path_main = os.path.join('C:\\','Users','szaro','Desktop','jupyter')

  scaler1 = joblib.load(os.path.join(path_main,'scaler','scaler_spektrogram')) if if_scaler==True else None  
  scaler2 = joblib.load(os.path.join(path_main,'scaler','scaler_mel_spektrogram')) if if_scaler==True else None  
  '''scaler3 = joblib.load('scaler_multitaper') if if_scaler==True else None '''
  scaler4 = joblib.load(os.path.join(path_main,'scaler','scaler_8_classic')) if if_scaler==True else None  
  scaler5 = joblib.load(os.path.join(path_main,'scaler','scaler_8_classic_plus_MIR')) if if_scaler==True else None  

  # k - kolejne nagrania w zbiorze 
  
  for k in range(nr_nagr_pocz, nr_nagr_pocz+ wielkosc_batcha): #np.shape(file_names_train_set)[0]): #np.shape(file_names_train_set)[0]): #np.shape(file_names_train_set)[0]):  # 87 dla zbioru train  len(file_names_train_set)

    ##### pętla do wycięgnięcia info o interesujących chunksach
    empty_list = np.array([[] for _ in range(np.shape(indices[k])[0])]) #np.array([[] for _ in range(np.shape(indices[k])[0])])
    # warunki: bo może się zdarzyć że macierz tablica pustych elementów będzie w tych dwóch 'chunks_species', 'call_id', i wtedy np.take nie działa 
    if not any(df_to_np[k][4]) and not any(df_to_np[k][5]):  
      info_chunksy[k] = [np.take(df_to_np[k][i],indices[k]) for i in [0,1,2,3]] + [empty_list, empty_list] + [np.take(df_to_np[k][i],indices[k]) for i in [6, 7]]

      print(np.shape(info_chunksy[k][5]), 'no calls at all')
    elif not any(df_to_np[k][5]):
      info_chunksy[k] = [np.take(df_to_np[k][i],indices[k]) for i in [0,1,2,3,4]] + [empty_list] + [np.take(df_to_np[k][i],indices[k]) for i in [6, 7]]                  
      print(np.shape(info_chunksy[k][5]), 'no calls of interest')
    else:
      info_chunksy[k] = [np.take(df_to_np[k][i],indices[k]) for i in [0,1,2,3,4,5,6,7]]

    start_time = time.time()
    if representation_type=='spektrogram':
        representation = np.empty([np.shape(indices[k])[0], 63, 148])
    if representation_type=='mel-spektrogram':
        representation = np.empty([np.shape(indices[k])[0], 60, 148])
    '''representation3 = np.empty([np.shape(indices[k])[0], 64, 149])'''
    if representation_type=='8_classic':
        representation = np.empty([np.shape(indices[k])[0], np.sum(summary_1d), 8])
    if representation_type=='8_classic_plus_MIR':
        representation = np.empty([np.shape(indices[k])[0], np.sum(summary_1d), 39])
   
    ##### a to już pętla do reprezentacji, pojedyncze chunksy
    for num, i in enumerate(indices[k]):                        # przykładowo 262 dla pierwszego nagrania w zbiorze train, bo tyle mamy tam chunksów
      
      # wczytanie chunka naszego półsekundowego:
      y, sr = librosa.load(path_wav + '/'+ file_names_train_set[k], sr = 44100, offset = result_dataframe_train['chunk_start'][k][i]/sr, duration = chunk_length_ms/1000)
      if representation_type=='spektrogram':   
      ####### reprezentacja 1  ------- 63 x 148 ------ SPEKTROGRAM
          representation[num] = my_spektrogram(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, scaler1)
      if representation_type=='mel-spektrogram': 
      ####### reprezentacja 2 (3 V3) - mel spektrogram  ------- 60 x 148 ------  
          representation[num] = my_melspektrogram(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, n_mels, scaler2)
        
      ####### reprezentacja 3 (5b) - multitaper o większej rozdzielczości ------- 64 x 149 ------
      # Ale tak naprawdę to nie jest multitaper, ale coś innego, nie wiem w sumie do końca co. Multitaper + Time-frequency reassignement spectrogram
      '''representation3[num] = my_multitaper(y, N, step, Np, K, tm, flock, tlock, f_min, f_max, scaler3) '''
      if representation_type=='8_classic': 
          representation[num] = my_8_classic(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, summary_1d, scaler4)
      if representation_type=='8_classic_plus_MIR': 
          representation[num] = my_8_classic_plus_MIR(y, n_fft, win_length, hop_length, window, sr, f_min, f_max, summary_1d, scaler5)

    repr_full[k] = representation
    print(k,'-', file_names_train_set[k], '- chunks:', np.shape(indices[k])[0], '- time:', time.time()-start_time)

  return [file_names_train_set , indices, info_chunksy, repr_full]     # repr_full3, representation_type
    <file_sep># -*- coding: utf-8 -*-
"""
Created on Tue May 25 12:59:30 2021

@author: Hania
 z best_model_from_random.py 
 
do sprawdzenia modelu wybranego z random searcha. Fit na trainie, early stopping dzięki walidowi. 

"""

import config
import numpy as np
from keras import backend as K
from keras.wrappers.scikit_learn import KerasClassifier
K.set_image_data_format('channels_last')
from sklearn.metrics import average_precision_score, make_scorer
from sklearn.metrics import roc_auc_score, average_precision_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import roc_curve
import matplotlib.pyplot as plt
#from config_agi import parse_model_config

from saving_utils import save_predictions, save_as_json, LoggerWrapper
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from useful_func_cnn import wczyt_dane_moje, make_cnn_model1, av_precision2, av_precision, make_cnn_bulbul, make_cnn_github2019, make_cnn_grid_corr, wczytaj_testy
#chosen_repr = 'spektrogram'
#chosen_cnn = 'bulbul' 
import os
import pickle
import time
import sys
import tensorflow as tf
import json

#import tensorflow as tf
from tensorflow.compat.v1.keras.backend import set_session
config2 = tf.compat.v1.ConfigProto()
config2.gpu_options.per_process_gpu_memory_fraction = 0.5
set_session(tf.compat.v1.Session(config=config2))

# python best_model_from_random.py reprezentacja link_najlepsze_par ile_epok
# python best_model_check.py spektrogram ..\results\grid\spektrogram\2021-05-13-00-05\best_params.json 50
# python best_model_check.py spektrogram ..\results\grid\spektrogram\2021-05-19-23-05\best_params.json 100

if __name__=='__main__':

    # Load configs
    chosen_repr = sys.argv[1] # 'spektrogram' #
    with open(sys.argv[2], 'r') as fp:
        best_params = json.load(fp)
    epochs = int(sys.argv[3] )
    exp_dir = os.path.join("..","results","PR", f"{chosen_repr}", time.strftime('%Y-%m-%d-%H-%M'))
    try:
        os.makedirs(exp_dir)
    except FileExistsError:
        pass
       
    
    start_time = time.time()
    logger_wrapper = LoggerWrapper(exp_dir)
    sys.stderr.write = logger_wrapper.log_errors
    logger_wrapper.logger.info(f'Running {sys.argv}')
        
    cv_split, dX, dy, di, dX_test, dy_test, di_test, cv, X_val, y_val, train_X, train_y = wczyt_dane_moje(chosen_repr, config, 'wlasc') 
    
    print(chosen_repr)
    if chosen_repr=="spektrogram":
        in_shape = (63, 148, 1)
    if chosen_repr=="mel-spektrogram":
        in_shape = (60, 148, 1)    
        
    
    dX_resh = dX.reshape(dX.shape[0], in_shape[0], in_shape[1], 1).astype('float32')     # trzeba wrócić z liniowego, które miało isć do podst. klasyf -> do obrazka 2d
    input_shape = (np.shape(dX_resh)[1], np.shape(dX_resh)[2], np.shape(dX_resh)[3]) # wielkoc obrazka, 1 kanał, na końcu podawany
    
    
    from tensorflow.python.client import device_lib
    print(device_lib.list_local_devices())

    
    print("Fit model on training data")
    y_val = y_val.astype(int)
    train_y = train_y.astype(int)
    
    X_val_resh = X_val.reshape(X_val.shape[0],in_shape[0], in_shape[1], 1).astype('float32') 
    train_X_resh = train_X.reshape(train_X.shape[0], in_shape[0], in_shape[1], 1).astype('float32')
    
    model = make_cnn_grid_corr(input_shape = best_params['input_shape'],
                                lr = best_params['lr'],
                                filters = best_params['filters'],
                                drop_out = best_params['drop_out'],
                                layers = best_params['layers'],
                                if_dropout = best_params['if_dropout'],
                                dense_layer_sizes = best_params['dense_layer_sizes']
                               # activation_function = best_params['activation_function']
                                ) #input_shape = best_params['input_shape'])
    
    my_callbacks = [
    tf.keras.callbacks.EarlyStopping(monitor="val_auc", mode="max", patience=3, restore_best_weights = True),
    ]

    
    result = model.fit(train_X_resh, train_y, validation_data = (X_val_resh,y_val), epochs=best_params['epochs'],
                    batch_size=best_params['batch_size'],callbacks = my_callbacks, verbose = 1)
    
    
# %%    
    
    # wczytanie testu
    testing_features, testing_target = wczytaj_testy(chosen_repr)    
    testing_features_resh = testing_features.reshape(testing_features.shape[0], in_shape[0], in_shape[1], 1).astype('float32')
    
    # zapis predykcji
    save_predictions(dX_resh, dy, cv_split, testing_features_resh, testing_target, model, exp_dir)
    y_preds = model.predict_proba(testing_features_resh) 
    # %%  
    # serialize weights to HDF5
    model.save(os.path.join(exp_dir,"my-model.h5"))
  
    print("Saved model to disk")
    
    all_scores = {}
    all_scores['test_AUC_PR'] = average_precision_score(testing_target, y_preds)  #scorer(model, dX_test, dy_test)
    all_scores['test_AUC_ROC'] = roc_auc_score(testing_target, y_preds)
    
    
    print('Z GRIDA: ',all_scores)
    
    
    logger_wrapper.logger.info('z grida')
    logger_wrapper.logger.info(all_scores)
    
       
    # zapisanie historii, by dało się wyrysować krzywe uczenia w przyszłosci , loss i pr_auc
    with open(os.path.join(exp_dir,'history.pckl'), 'wb') as file_pi:
        pickle.dump(result.history, file_pi)
    # jak wczytać:   
    #f = open('history.pckl', 'rb')
    # = pickle.load(f)
    #f.close()
   
    model_json = (model.to_json(indent=0))
    with open(os.path.join(exp_dir,"architecture-only-refit.json"), "w") as json_file:
        json_file.write(model_json)

    # wyrys krzywych roc auc i pr auc 
    prec_nb, recall_nb, _ = precision_recall_curve(testing_target, y_preds)
    fpr_nb, tpr_nb, _ = roc_curve(testing_target, y_preds)
    fig = plt.figure(figsize=(10,10))
    plt.rcParams.update({'font.size': 22})
    plt.title('AUC PR')
    plt.plot(recall_nb, prec_nb,  'b', label = 'CNN AUC = %0.2f%% ' % (100*average_precision_score(testing_target, y_preds)))
    plt.legend(loc = 'upper right')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('Precyzja (Precision)')
    plt.xlabel('Czułość (Recall)')
    #plt.show()
    fig.savefig(os.path.join(exp_dir,'AUC-PR-refit'+ time.strftime('%Y-%m-%d-%H-%M')))
    
    fig = plt.figure(figsize=(10,10))
    plt.rcParams.update({'font.size': 22})
    plt.title('AUC ROC')
    plt.plot(fpr_nb, tpr_nb, 'b', label = 'CNN AUC = %0.2f%%' % (100*roc_auc_score(testing_target, y_preds)))
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('Wskaźnik czułości (True Positive Rate)')
    plt.xlabel('Odsetek fałszywie pozytywnych alarmów (False Positive Rate)')
    #plt.show()
    fig.savefig(os.path.join(exp_dir,'AUC-ROC-refit'+ time.strftime('%Y-%m-%d-%H-%M')))

    # wyrys krzywych uczenia dla loss i pr auc
    # summarize history for loss
    fig = plt.figure(figsize=(10,10))
    plt.plot(result.history['loss'])
    plt.plot(result.history['val_loss'])
    plt.title('Model loss')
    plt.ylabel('Koszt (loss)')
    plt.xlabel('Liczba epok')
    plt.legend(['trening','walidacja'], loc='upper right')
   # plt.show()
    fig.savefig(os.path.join(exp_dir,'uczenie-loss-refit'+ time.strftime('%Y-%m-%d-%H-%M')))
    
    #auc_nr = 'auc_'+str(int(n_iter) + 1)
    
    fig = plt.figure(figsize=(10,10))
    plt.plot(result.history['auc'])
    plt.plot(result.history['val_auc'])
    plt.title('Model PR AUC')
    plt.ylabel('Pole pod krzywą PR')
    plt.xlabel('Liczba epok')
    plt.legend(['trening','walidacja'], loc='lower right')
    #plt.show()
    fig.savefig(os.path.join(exp_dir,'uczenie-PR-AUC-refit' + time.strftime('%Y-%m-%d-%H-%M')))
    
    <file_sep>folder na pliki lokalne pythonowe
<file_sep># -*- coding: utf-8 -*-
"""
Created on Thu Apr 22 13:54:13 2021

@author: Hania
"""

from sklearn.metrics import average_precision_score
from keras import backend as K
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten #Activation, 
from keras.layers import Conv2D, MaxPooling2D
from  sklearn.model_selection import PredefinedSplit
import keras

def av_precision(y_true, y_pred, normalize = False):
    #return tf.py_function(average_precision_score, (y_true, y_pred), tf.float64)   
    #return tf.convert_to_tensor(average_precision_score(y_true, y_pred), dtype=tf.float32)

#def sk_pr_auc(y_true, y_pred):
    return tf.py_function(average_precision_score, (y_true, y_pred), tf.float64)

# =============================================================================
# def av_precision3(y_true, y_pred, normalize = False):
# 
#     y_true = y_true.reshape((-1))
#     y_pred = y_pred.reshape((-1))
#     return tf.py_function(average_precision_score, (y_true, y_pred), tf.float64)
# 
# =============================================================================
def av_precision2(y_true, y_pred, normalize = False):
    return average_precision_score(y_true, y_pred)

def make_cnn_model1(input_shape = (63, 148, 1),  dense_layer_sizes=128, filters=10, kernel_size=(3,3),
                pool_size=(2,2), drop_out=0.5):
  
# =============================================================================
#     dense_layer_sizes=128
#     filters=10
#     kernel_size=(3,3) #(3,3)
#     pool_size= (2,2) 
#     lr=0.0001 
#     drop_out = 0.5
# =============================================================================
    #import tensorflow
    #from tensorflow import keras
    from keras.models import Sequential
    from keras.layers import Dense, Dropout, Flatten #Activation, 
    from keras.layers import Conv2D, MaxPooling2D
    model = Sequential()
    model.add(Conv2D(filters, kernel_size,input_shape=input_shape, activation='relu'))
    model.add(MaxPooling2D(pool_size=pool_size))
    
    model.add(Conv2D(filters, kernel_size, activation='relu'))
    model.add(MaxPooling2D(pool_size=pool_size))
        
    
    model.add(Flatten())
    model.add(Dense(dense_layer_sizes, activation='relu'))
    model.add(Dropout(drop_out))
    model.add(Dense(32, activation='relu'))
    model.add(Dropout(drop_out))
    model.add(Dense(1, activation='sigmoid'))
    
    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=[tf.keras.metrics.AUC(curve='PR')])#  [av_precision]

    return model


def make_cnn_grid(input_shape = (63, 148, 1),  lr=0.001, filters = 10, drop_out = 0.5, layers = 3,
                  dense_layer_sizes=256, activation_function = 'relu' ):
  
# =============================================================================
#     'lr': [0.0005, 0.001, 0.01, 0.1],
#     'decay': [0, 0.01],
#     'filters': [10, 16, 20],
#     #kernel_size = [(3,3), (5,5)]
#     #pool_size = [(2,2), (3,3)]
#     'dropout': [0, 0.1, 0.35, 0.5],
#     'batch_size': [32, 64, 128],
#     'epochs': [10, 20, 30, 50],
#     'dense_layer_sizes': [128, 256],
#     'activation_function': ['LeakyReLU', 'ReLU']
# 
# =============================================================================
    kernel_size = (3,3)
    pool_size = (2,2)
    
    from keras.models import Sequential
    from keras.layers import Dense, Dropout, Flatten #Activation, 
    from keras.layers import Conv2D, MaxPooling2D
    model = Sequential()
    model.add(Conv2D(filters, kernel_size,input_shape=input_shape, activation=activation_function))
    model.add(MaxPooling2D(pool_size=pool_size))
    
    for i in range(layers-1):
        model.add(Conv2D(filters, kernel_size, activation=activation_function))
        model.add(MaxPooling2D(pool_size=pool_size))
       
    model.add(Flatten())
    model.add(Dense(dense_layer_sizes, activation=activation_function))
    model.add(Dropout(drop_out))
    model.add(Dense(32, activation=activation_function))
    model.add(Dropout(drop_out))
    model.add(Dense(1, activation='sigmoid'))
    
    model.compile(loss='binary_crossentropy',
                  optimizer=keras.optimizers.Adam(learning_rate = lr),
                  metrics=[tf.keras.metrics.AUC(curve='PR')])#  [av_precision]

    return model

def make_cnn_grid_corr(input_shape = (63, 148, 1),  lr=0.001, filters = 10, drop_out = 0.5, layers = 3,
                  dense_layer_sizes=256, activation_function = 'relu', if_dropout = 0 ): # , if_scheduler = 0
  
# =============================================================================
#     'lr': [0.0005, 0.001, 0.01, 0.1],
#     'decay': [0, 0.01],
#     'filters': [10, 16, 20],
#     #kernel_size = [(3,3), (5,5)]
#     #pool_size = [(2,2), (3,3)]
#     'dropout': [0, 0.1, 0.35, 0.5],
#     'batch_size': [32, 64, 128],
#     'epochs': [10, 20, 30, 50],
#     'dense_layer_sizes': [128, 256],
#     'activation_function': ['LeakyReLU', 'ReLU']
# 
# =============================================================================
    kernel_size = (3,3)
    pool_size = (2,2)
    
    from keras.models import Sequential
    from keras.layers import Dense, Dropout, Flatten #Activation, 
    from keras.layers import Conv2D, MaxPooling2D, SpatialDropout2D
    model = Sequential()
    model.add(Conv2D(filters, kernel_size,input_shape=input_shape, activation=activation_function))
    model.add(MaxPooling2D(pool_size=pool_size))
    model.add(SpatialDropout2D(if_dropout))
    
    for i in range(layers-1):
        model.add(Conv2D(filters, kernel_size, activation=activation_function))
        model.add(MaxPooling2D(pool_size=pool_size))
        model.add(SpatialDropout2D(if_dropout))
       
    model.add(Flatten())
    model.add(Dense(dense_layer_sizes, activation=activation_function))
    model.add(Dropout(drop_out))
    model.add(Dense(32, activation=activation_function))
    model.add(Dropout(drop_out))
    model.add(Dense(1, activation='sigmoid'))
    
# =============================================================================
#     if if_scheduler==1:
#         step = tf.Variable(0, trainable=False)
#         boundaries = [160000]
#         values = [lr, lr*0.1]
#         learning_rate_fn = keras.optimizers.schedules.PiecewiseConstantDecay(
#             boundaries, values)
#         
#         # Later, whenever we perform an optimization step, we pass in the step.
#         #learning_rate = learning_rate_fn(step)
#         optimizer=keras.optimizers.Adam(learning_rate = learning_rate_fn)
#     else:
#         optimizer=keras.optimizers.Adam(learning_rate = lr)
#     
# =============================================================================
    
    model.compile(loss='binary_crossentropy',
                  optimizer=keras.optimizers.Adam(learning_rate = lr),
                  metrics=[tf.keras.metrics.AUC(curve='PR')])#  [av_precision]

    return model



def make_cnn_bulbul(input_shape = (63, 148, 1),  drop_out=0.5):
  
 # dense_layer_sizes=128, filters=10, kernel_size=(3,3), pool_size=(2,2),   
# =============================================================================
#     dense_layer_sizes=128
#     filters=10
#     kernel_size=(3,3) #(3,3)
#     pool_size= (2,2) 
#     lr=0.0001 
#     drop_out = 0.5
# =============================================================================
    #import tensorflow
    #from tensorflow import keras

    model = Sequential()
    
    model.add(Conv2D(16, kernel_size=(3,3),input_shape=input_shape, activation='relu'))
    model.add(MaxPooling2D(pool_size=(3,3)))
    
    model.add(Conv2D(16, kernel_size=(3,3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(3,3)))
    
    model.add(Conv2D(16, kernel_size=(1,3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(1,3)))
    
    #model.add(Conv2D(16, kernel_size=(1,3), activation='relu'))
    #model.add(MaxPooling2D(pool_size=(1,3)))
        
    
    model.add(Flatten())
    model.add(Dropout(drop_out))
    model.add(Dense(256, activation='relu'))
    model.add(Dropout(drop_out))
    model.add(Dense(32, activation='relu'))
    model.add(Dropout(drop_out))
    model.add(Dense(1, activation='sigmoid'))
    
    model.compile(loss='binary_crossentropy',
                  optimizer='adam',
                  metrics=[tf.keras.metrics.AUC(curve='PR')])#  [av_precision] daje nany w validzie

    return model


def make_cnn_github2019(input_shape = (63, 148, 1), dense_layer_sizes=128, filters=10, kernel_size=(3,3),
                pool_size=(2,2), drop_out=0.5):
    # https://github.com/hannape/CNN-second/blob/master/CNN_for_new_data.ipynb
    import functools
    from functools import partial, update_wrapper
    dense_layer_sizes=128
    filters=10
    kernel_size=(3,3) #(3,3)
    pool_size= (2,2) 
    #lr=0.0001 
    drop_out = 0.5
    
# ========= nie działa, jakie stare wersje kerasów, tfów i pythonów =================================
    def wrapped_partial(func, *args, **kwargs):
     	partial_func = partial(func, *args, **kwargs)
     	update_wrapper(partial_func, func)
     	return partial_func

    def binary_crossentropy_weigted(y_true, y_pred, class_weights):
     	y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon())
     	loss = K.mean(class_weights*(-y_true * K.log(y_pred) - (1.0 - y_true) * K.log(1.0 - y_pred)),axis=-1)
     	return loss
    
    custom_loss = wrapped_partial(binary_crossentropy_weigted, class_weights=np.array([0.02, 0.98])) ## scoring for model.compile
    
# =====https://datascience.stackexchange.com/questions/58735/weighted-binary-cross-entropy-loss-keras-implementation
           
    def weighted_bce(y_true, y_pred):
        weights = (y_true * 49) + 1
        bce = K.binary_crossentropy(y_true, y_pred)
        weighted_bce = K.mean(bce * weights)
        return weighted_bce
        
    
    model = Sequential()
    model.add(Conv2D(filters, kernel_size,input_shape=input_shape, activation='relu'))
   # model.add(Conv2D(16, (3, 3), input_shape=input_shape ))

    model.add(MaxPooling2D(pool_size=pool_size))
    model.add(Conv2D(filters, kernel_size, activation='relu'))
    model.add(MaxPooling2D(pool_size=pool_size))
    model.add(Conv2D(filters, kernel_size, activation='relu'))
    model.add(MaxPooling2D(pool_size=pool_size))
    
    
    model.add(Flatten())
    model.add(Dense(dense_layer_sizes, activation='relu'))
    model.add(Dropout(drop_out))
    model.add(Dense(32, activation='relu'))
    model.add(Dropout(drop_out))
    model.add(Dense(1, activation='sigmoid'))
    
    model.compile(loss= 'binary_crossentropy', #weighted_bce, #'binary_crossentropy',
                  optimizer='adam',#keras.optimizers.Adam(lr),#'adam',
                  metrics=['accuracy', tf.keras.metrics.AUC(curve='PR'), av_precision])

    return model




def wczyt_dane_moje(chosen_repr, config, set_type):
    import pandas as pd
    import os
    from useful_stuff import  make_test
    
    if set_type == 'wlasc':
        print("dane wlasciwe")
        df1 = pd.read_hdf(os.path.join('..','..','jupyter','data','df_train_'+ str(chosen_repr) +'_norm.h5'),'repr' + \
                          str((config.representation_2d + [''] + config.representation_1d).index(chosen_repr)+1))
    
        df2 = pd.read_hdf(os.path.join('..','..','jupyter','data','df_valid_norm.h5'),'df')
        all_repr = config.representation_1d + config.representation_2d
        all_repr.remove(chosen_repr)
        df2 = df2.drop(all_repr, axis=1)
        cv_split, dX, dy, di, cv = make_cv_split_cnn(df1, df2, chosen_repr)    
        print("Własciwy train+walid wczytany")
        
        valid = df2.loc[:,['rec_name', 'chunk_ids', chosen_repr, 'has_bird']].values
        train = df1.loc[:,['rec_name', 'chunk_ids', chosen_repr, 'has_bird']].values
        #valid_identifiers = valid[:, 0:2]
        valid_X = np.stack(valid[:, 2])
        valid_y = valid[:, 3]
        train_X = np.stack(train[:, 2])
        train_y = train[:, 3]
        
    
    if set_type == 'testy': 
        print("dane mini, do testow")
        df = pd.read_hdf(os.path.join('..','..','jupyter','data','df_val300_norm.h5'),'val')
        all_repr = config.representation_1d + config.representation_2d
        all_repr.remove(chosen_repr)
        df = df.drop(all_repr, axis=1)
        df1 = df[50:300]#df.sample(n=250, random_state = 667)
        df2 = df[0:50]#df.sample(n=50, random_state = 667)
        cv_split, dX, dy, di, cv = make_cv_split_cnn(df1, df2, chosen_repr) 
        print("Testowy train+walid wczytany")
        valid_X = []
        valid_y = []
        
    df3 = pd.read_hdf(os.path.join('..','..','jupyter','data','df_test_3rec_'+ str(chosen_repr) +'_norm.h5'),'df')
    dX_test, dy_test, di_test = make_test(df3, chosen_repr)
    #print("Test wczytany")
    
    return cv_split, dX, dy, di, dX_test, dy_test, di_test, cv, valid_X, valid_y, train_X, train_y


def make_cv_split_cnn(train, valid, chosen_repr, classifier=False):   
    
    # 
    train = train.loc[:,['rec_name', 'chunk_ids', chosen_repr, 'has_bird']].values
    train_identifiers = train[:, 0:2]
    train_X = np.stack(train[:, 2])
    train_y = train[:, 3]
    
    valid = valid.loc[:,['rec_name', 'chunk_ids', chosen_repr, 'has_bird']].values
    valid_identifiers = valid[:, 0:2]
    valid_X = np.stack(valid[:, 2])
    valid_y = valid[:, 3]

    dX = np.vstack((train_X, valid_X))
    print('min train+walid: ', np.amin(dX))
    dy = np.hstack((train_y, valid_y))
    di = np.vstack((train_identifiers, valid_identifiers))
    
    train_indices = np.array(range(0, train_X.shape[0]))
    val_indices = np.array(range(train_X.shape[0], dX.shape[0]))
    
    cv_split = [(train_indices, val_indices)] # ,(train_indices, val_indices)
    
    dX = np.reshape(dX, newshape=(dX.shape[0],-1))
    dy = dy.astype(int)
   
    # http://www.wellformedness.com/blog/using-a-fixed-training-development-test-split-in-sklearn/
    test_fold = np.concatenate([
    # The training data.
    np.full( train_X.shape[0],-1, dtype=np.int8),
    # The development data.
    np.zeros(valid_X.shape[0], dtype=np.int8)
    ])
    cv = PredefinedSplit(test_fold)

    print(test_fold)
    return cv_split, dX, dy, di, cv


def wczytaj_testy(chosen_repr):
    # najpierw wczyt podziału całosci
    from useful_stuff import make_cv_split, make_test  
    from useful_stuff import read_best_models_8_classic, read_best_models_8_classic_plus_MIR
    import pandas as pd
    import os
    import config
    from sklearn.metrics import roc_auc_score, average_precision_score
    from wczytanie_danych import funkcja_wczytanie_danych
    from reprezentacje import funkcja_reprezentacja
    import numpy as np
    import pandas as pd
    import json   
    import glob
    import time 
    
    start_time = time.time()
    print(config.path_test1618_txt)
    path_main = os.path.join('C:\\','Users','szaro','Desktop','jupyter')
    print(path_main)
    path_test1618_txt = os.path.join(path_main, config.path_test1618_txt)
    path_train161718_txt = os.path.join(path_main, config.path_train161718_txt)
    path_test1618_wav = os.path.join(path_main, config.path_test1618_wav)
    path_train161718_wav = os.path.join(path_main, config.path_train161718_wav)
    
    test_new_only = 1
    _,_,_, data_test_new = funkcja_wczytanie_danych(path_test1618_txt, path_train161718_txt, path_test1618_wav,\
                                                                                     path_train161718_wav, config.balance_types, config.balance_ratios, config.chunk_length_ms,\
                                                                                     config.chunk_overlap, config.calls_0, config.calls_1, config.calls_unknown, config.tolerance, config.valid_set,\
                                                                                     config.test_rec_to_cut, config.columns_dataframe, test_new_only)
    print("--- Funkcja wczytanie danych: %s sekund ---" % (time.time() - start_time))
    #  Wczytanie testowych reprezentacji
    import numpy as np
    
    start_time = time.time()
    np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
    
    
    ## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    if_scaler = True 
    #chosen_repr = '8_classic'
    nr_nagr_pocz = 0
    wielkosc_batcha = 18
    ## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    
    
    ## repr_full3, 
    file_names, indices, info_chunksy, repr_full_scaled = funkcja_reprezentacja(path_test1618_wav, data_test_new, chosen_repr, if_scaler, \
                                                                   config.repr_1d_summary, config.summary_1d, config.sr, config.chunk_length_ms, \
                                                                   config.n_fft, config.win_length, config.hop_length, config.window, config.f_min, config.f_max,\
                                                                   config.n_mels, config.N, config.step, config.Np, config.K, config.tm, \
                                                                   config.flock, config.tlock, nr_nagr_pocz, wielkosc_batcha)
    #file_names, indices, info_chunksy, representation_type, repr_full = funkcja_reprezentacje(path_train161718_wav, data_train, '8_classic', repr_1d_summary, summary_1d, sr, chunk_length_ms, \
    #                          n_fft, win_length, hop_length, window, f_min, f_max, n_mels, N, step, Np, K, tm, flock, tlock)
    
    print("--- Funkcja reprezentacja danych: %s minut ---" % ((time.time() - start_time)/60))
    
    
    file_names_list, chunk_ids_list, has_bird_list, representation_list = [],[],[],[]
    
    
    for num,i in enumerate(indices[nr_nagr_pocz:nr_nagr_pocz+wielkosc_batcha]):
    
      file_names_list.extend([file_names[num] for i in range(len(i))])
      chunk_ids_list.extend((info_chunksy[num][0]))  #.tolist()
      has_bird_list.extend((info_chunksy[num][3]))
      representation_list.extend([repr_full_scaled[num][i] for i in range(len(i))]) # cała reprezentacja1
      
      
    testing_features = np.array(representation_list) #pd.DataFrame(data = representation_list, columns =str(chosen_repr))
    testing_target = has_bird_list
    
    testing_features = np.reshape(testing_features, newshape=(testing_features.shape[0],-1))
    
    print("Test wczytany 18 nagran")

    return testing_features, testing_target




"""This file is part of the TPOT library.
TPOT was primarily developed at the University of Pennsylvania by:
    - <NAME> (<EMAIL>)
    - <NAME> (<EMAIL>)
    - <NAME> (<EMAIL>)
    - and many more generous open source contributors
TPOT is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3 of
the License, or (at your option) any later version.
TPOT is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with TPOT. If not, see <http://www.gnu.org/licenses/>.
"""

import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.utils import check_array


class ZeroCount(BaseEstimator, TransformerMixin):
    """Adds the count of zeros and count of non-zeros per sample as features."""

    def fit(self, X, y=None):
        """Dummy function to fit in with the sklearn API."""
        return self

    def transform(self, X, y=None):
        """Transform data by adding two virtual features.
        Parameters
        ----------
        X: numpy ndarray, {n_samples, n_components}
            New data, where n_samples is the number of samples and n_components
            is the number of components.
        y: None
            Unused
        Returns
        -------
        X_transformed: array-like, shape (n_samples, n_features)
            The transformed feature set
        """
        X = check_array(X)
        n_features = X.shape[1]

        X_transformed = np.copy(X)

        non_zero_vector = np.count_nonzero(X_transformed, axis=1)
        non_zero = np.reshape(non_zero_vector, (-1, 1))
        zero_col = np.reshape(n_features - non_zero_vector, (-1, 1))

        X_transformed = np.hstack((non_zero, X_transformed))
        X_transformed = np.hstack((zero_col, X_transformed))

        return X_transformed<file_sep># -*- coding: utf-8 -*-
"""
Created on Tue May 18 11:55:04 2021

@author: Hania
"""

# -*- coding: utf-8 -*-
"""
Created on Thu May  6 15:58:03 2021

@author: Hania

Sprawdzenie grida, zmiany po rozmowie z Agą 13.05.21

"""



import config
import numpy as np
from keras import backend as K
from keras.wrappers.scikit_learn import KerasClassifier
K.set_image_data_format('channels_last')
from sklearn.metrics import average_precision_score, make_scorer
from sklearn.metrics import roc_auc_score, average_precision_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import roc_curve
import matplotlib.pyplot as plt
#from config_agi import parse_model_config

from saving_utils import save_predictions, save_as_json, LoggerWrapper
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from useful_func_cnn import wczyt_dane_moje, make_cnn_model1, av_precision2, av_precision, make_cnn_grid_corr, wczytaj_testy
#chosen_repr = 'spektrogram'
#chosen_cnn = 'bulbul' 
import os
import pickle
import time
import sys
import tensorflow as tf
import json
import pandas as pd

# cos żeby się GPU nie dławiło: 
# =============================================================================
from tensorflow.compat.v1.keras.backend import set_session
config2 = tf.compat.v1.ConfigProto()
config2.gpu_options.per_process_gpu_memory_fraction = 0.3 #0.5
set_session(tf.compat.v1.Session(config=config2))

# =============================================================================
# =============================================================================
# config3 = tf.compat.v1.ConfigProto()
# config3.gpu_options.allow_growth=True
# set_session(tf.compat.v1.Session(config=config3))
# =============================================================================


#%%


# python main_04052021_cnny.py spektrogram bulbul batch_size epochs
# python main_06052021_grid_cnny_corr.py spektrogram 100

if __name__=='__main__':

    # Load configs
    chosen_repr = sys.argv[1] # 'spektrogram' #
    n_iter = sys.argv[2]


    exp_dir = os.path.join("..","results","grid", f"{chosen_repr}", time.strftime('%Y-%m-%d-%H-%M'))
    try:
        os.makedirs(exp_dir)
    except FileExistsError:
        pass
       
    
    start_time = time.time()
    logger_wrapper = LoggerWrapper(exp_dir)
    sys.stderr.write = logger_wrapper.log_errors
    logger_wrapper.logger.info(f'Running {sys.argv}')
    
    # cv - podział do randomizedsearcha, by nie robił mi CV. dX - train + valid. Oprócz tego są też osobno jako X_val i train_X
    cv_split, dX, dy, di, dX_test, dy_test, di_test, cv, X_val, y_val, train_X, train_y = wczyt_dane_moje(chosen_repr, config, 'wlasc') 
    
    print(chosen_repr)
    if chosen_repr=="spektrogram":
        in_shape = (63, 148, 1)
    if chosen_repr=="mel-spektrogram":
        in_shape = (60, 148, 1)    
        
    
    dX_resh = dX.reshape(dX.shape[0], in_shape[0], in_shape[1], 1).astype('float32')     # trzeba wrócić z liniowego, które miało isć do podst. klasyf -> do obrazka 2d
    input_shape = (np.shape(dX_resh)[1], np.shape(dX_resh)[2], np.shape(dX_resh)[3]) # wielkoc obrazka, 1 kanał, na końcu podawany
    X_val_resh = X_val.reshape(X_val.shape[0], in_shape[0], in_shape[1], 1).astype('float32') 
    
    # upewnienie sie że działam na GPU
    #from tensorflow.python.client import device_lib
    #print(device_lib.list_local_devices())
   
    
    params = {
    'lr': [0.0001, 0.0005, 0.001, 0.005], # początkowy LR do Adama 0.01 za duży jest
    'input_shape': [in_shape],
    'filters': [10, 25, 50, 100, 150],
    'layers': [3, 4],  # ilosć warstw [konwolucja +pooling] 3, 4, 5,
    'drop_out': [0, 0.1, 0.2, 0.3, 0.4, 0.5],  # dropout po dwóch warstwach gęstych
    'if_dropout': [0, 0.1, 0.2], # parametry do spatialDropout2D. dropout po poolingach.
    # Internety mówią że rzadziej się daje dropout po conv,a jesli już to małe wartoci
    'batch_size': [16, 32, 64, 128],
    'epochs': [100],#[5, 8, 10, 15, 20, 30, 50],
    #'padding': ["same"],  # "valid",  "same" - nie ma sensu, bo to pooling najbardziej zmniejsza reprezentację, nie conv i padding.
    # Trzeba by zmienić archotekturę i wrzucać więcj conv, anie na przemian conv i pool
    'dense_layer_sizes': [128, 256], # rozmiar pierwszej warstwy gęstej
    #'if_scheduler': [0, 1]
    }
    
       
     
    model = KerasClassifier(build_fn = make_cnn_grid_corr, verbose=1)
    logger_wrapper.logger.info('macierz badanych parametrów')
    logger_wrapper.logger.info(params)
    
    with open(os.path.join(exp_dir,"matrix_params.json"), "w") as json_file:
        json.dump(params, json_file)  # json_file.write(params.to_json(indent=2))    
    
    my_callbacks = [
             tf.keras.callbacks.EarlyStopping(monitor="val_loss",patience=5, restore_best_weights = True),
         ]


    grid = RandomizedSearchCV(estimator=model, param_distributions=params, scoring = make_scorer(average_precision_score),
                        cv = cv, n_jobs=1, n_iter=int(n_iter), verbose = 3, random_state =667, refit = False) #
     # nie refitujemy, bo wtedy valid będzie i w treningu i w walidzie, więć early stopping nie będzie miał sensu
    grid_result = grid.fit(dX_resh, dy, validation_data=(X_val_resh, y_val.astype(np.float32)), callbacks = my_callbacks)
   
    
    print(grid_result.best_params_)
# =============================================================================
#   Tak było wczesniej, bez early stoppingu.
  
#     grid = RandomizedSearchCV(estimator=model, param_distributions=params, scoring = make_scorer(average_precision_score),
#                         cv = cv, n_jobs=1, n_iter=int(n_iter), verbose = 3, random_state =667) #
#     grid_result = grid.fit(dX_resh, dy)
#     
# =============================================================================
    
    logger_wrapper.logger.info('grid search trwał (s): ')
    logger_wrapper.logger.info(time.time() - start_time)
    
    best_params = grid_result.best_params_
    
    with open(os.path.join(exp_dir,"best_params.json"), "w") as json_file:
        json.dump(best_params, json_file, indent=1)    
    
        
    pd.DataFrame(grid_result.cv_results_['params']).to_pickle(os.path.join(exp_dir,"all_models.pkl")) 
     
    pd.DataFrame(grid_result.cv_results_['split0_test_score']).to_pickle(os.path.join(exp_dir,"all_models_scores.pkl") )
    
    
    def display_cv_results(search_results):
        print('Best score = {:.4f} using {}'.format(search_results.best_score_, search_results.best_params_))
        means = search_results.cv_results_['mean_test_score']
        stds = search_results.cv_results_['std_test_score']
        params = search_results.cv_results_['params']
        for mean, stdev, param in zip(means, stds, params):
            print('mean test accuracy +/- std = {:.4f} +/- {:.4f} with: {}'.format(mean, stdev, param))    
    
    display_cv_results(grid_result)   
    
    

        
    # odczyt:  pd.read_pickle("best20.pkl") 
    logger_wrapper.logger.info('All params:')
    logger_wrapper.logger.info(params)
    logger_wrapper.logger.info('Best params:', best_params)
    logger_wrapper.logger.info( best_params)
    logger_wrapper.logger.info(grid_result.cv_results_)
    
    
    # trzeba zrefitować na najlepszym zestawie, bo 
    #" This RandomizedSearchCV instance was initialized with refit=False. predict_proba is available only after refitting on the best parameters.
    # You can refit an estimator manually using the ``best_params_`` attribute" 
    # refit - ale trening dalej na trainie, a nie na całosci train + walid
    train_X_resh = train_X.reshape(train_X.shape[0], in_shape[0], in_shape[1], 1).astype('float32')
    
    model_refit = make_cnn_grid_corr(input_shape = best_params['input_shape'],
                            lr = best_params['lr'],
                            filters = best_params['filters'],
                            drop_out = best_params['drop_out'],
                            layers = best_params['layers'],
                            if_dropout = best_params['if_dropout'],
                            dense_layer_sizes = best_params['dense_layer_sizes']
                            )
    hist = model_refit.fit(train_X_resh, train_y.astype(np.float32), validation_data=(X_val_resh, y_val.astype(np.float32)), callbacks = my_callbacks,
                                   epochs=best_params['epochs'], batch_size=best_params['batch_size'])
    
    
     # zapisz model
    model_refit.save(os.path.join(exp_dir,"my-model.h5"))
    #grid_result.best_estimator_.model.save(os.path.join(exp_dir,"my-model.h5"))    
    print("Saved model to disk")
    
    # wczytanie testu
    testing_features, testing_target = wczytaj_testy(chosen_repr)  
    testing_features_resh = testing_features.reshape(testing_features.shape[0], in_shape[0], in_shape[1], 1).astype('float32')
    
    # zapis predykcji
    save_predictions(dX_resh, dy, cv_split, testing_features_resh, testing_target, model_refit, exp_dir)
    
    y_preds = model_refit.predict_proba(testing_features_resh) 

    # zapisanie historii, by dało się wyrysować krzywe uczenia w przyszłosci , loss i pr_auc
    with open(os.path.join(exp_dir,'history-z-refita.pckl'), 'wb') as file_pi:
        pickle.dump(hist.history, file_pi)
    # jak wczytać:   
    #f = open('history.pckl', 'rb')
    # = pickle.load(f)
    #f.close()
 # %%     
 
    model_json = (model_refit.to_json(indent=2))
    with open(os.path.join(exp_dir,"architecture-only-refit.json"), "w") as json_file:
        json_file.write(model_json)
        
 # %%    
    prec_nb, recall_nb, _ = precision_recall_curve(testing_target, y_preds)
    fpr_nb, tpr_nb, _ = roc_curve(testing_target, y_preds)
    print('AUC PR: ', average_precision_score(testing_target, y_preds))
    
    fig = plt.figure(figsize=(10,10))
    plt.rcParams.update({'font.size': 22})
    plt.title('AUC PR')
    plt.plot(recall_nb, prec_nb,  'b', label = 'CNN AUC = %0.2f%% ' % (100*average_precision_score(testing_target, y_preds)))
    plt.legend(loc = 'upper right')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('Precyzja (Precision)')
    plt.xlabel('Czułość (Recall)')
    #plt.show()
    fig.savefig(os.path.join(exp_dir,'AUC-PR-refit'+ time.strftime('%Y-%m-%d-%H-%M')))
    
    fig = plt.figure(figsize=(10,10))
    plt.rcParams.update({'font.size': 22})
    plt.title('AUC ROC')
    plt.plot(fpr_nb, tpr_nb, 'b', label = 'CNN AUC = %0.2f%%' % (100*roc_auc_score(testing_target, y_preds)))
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('Wskaźnik czułości (True Positive Rate)')
    plt.xlabel('Odsetek fałszywie pozytywnych alarmów (False Positive Rate)')
    #plt.show()
    fig.savefig(os.path.join(exp_dir,'AUC-ROC-refit'+ time.strftime('%Y-%m-%d-%H-%M')))
    

    
    # summarize history for loss
    fig = plt.figure(figsize=(10,10))
    plt.plot(hist.history['loss'])
    plt.plot(hist.history['val_loss'])
    plt.title('Model loss')
    plt.ylabel('Koszt (loss)')
    plt.xlabel('Liczba epok')
    plt.legend(['trening','walid'], loc='upper right')
   # plt.show()
    fig.savefig(os.path.join(exp_dir,'uczenie-loss-refit'+ time.strftime('%Y-%m-%d-%H-%M')))
    
    auc_nr = 'auc_'+str(int(n_iter) )
    val_auc_nr = 'val_auc_'+str(int(n_iter) )
    
    fig = plt.figure(figsize=(10,10))
    plt.plot(hist.history[auc_nr])
    plt.plot(hist.history[val_auc_nr])
    plt.title('Model PR AUC')
    plt.ylabel('Pole pod krzywą PR')
    plt.xlabel('Liczba epok')
    plt.legend(['trening','walid'], loc='lower right')
    #plt.show()
    fig.savefig(os.path.join(exp_dir,'uczenie-PR-AUC-refit' + time.strftime('%Y-%m-%d-%H-%M')))
    
    <file_sep># Gruba-kreska
Zebranie wszystkiego porządnie.

1. Poprawienie danych
2. Wczytanie danych
3. Main wczytanie i reprezentacje 
...
