# %% [markdown]
# # Exporatory analysis of features extracted from swine audios
# This dataset contains the means of the extracted features, for each audio
# Many descriptions of features were taken from the following books:
# - https://books.google.com/books?hl=en&lr=&id=AF30yR41GIAC&oi=fnd&pg=PP9&dq=Signal+Processing+Methods+for+Music+Transcription&ots=Ooq77iPfKD&sig=WFyZTyVVFtagCBGCRLWpV8rY-Tk  # noqa: E501

# %% [markdown]
# # Imports

# %%
import matplotlib.pyplot as plt
import librosa as lr
import librosa.display as lrdisp
import numpy as np
import pandas as pd
import seaborn as sns

FILE_PREFIX = 'kdd'

# %% [markdown]
# # Loading dataset

# %%
df = pd.read_csv('features_means.csv', index_col=0, verbose=True)
df.index = pd.to_datetime(df.index)
df['rac'] = False
df.loc['2020-09-22':, 'rac'] = True  # type: ignore

# %% [markdown]
# ## Checking for and dropping duplicates

# %%
# Resetting index for duplicate analysis
df.reset_index(inplace=True)
print("Duplicates by filename:",
      df.duplicated(subset=['file_name']).value_counts(),
      sep='\n')
df.drop_duplicates(subset=['file_name'], inplace=True)
print("Duplicates by (datetime, ala, grupo):",
      df.duplicated(subset=['datetime', 'ala', 'grupo']).value_counts(),
      sep='\n')
df.drop_duplicates(subset=['datetime', 'ala', 'grupo'], inplace=True)
# Rebuilding dataframe index
df.set_index('datetime', inplace=True)

# %%
print(df.shape)

# %% [markdown]
# ## Visualizing distribution of sample dates

# %%
df_tmp = pd.DataFrame(df['file_name'].resample('1D').count())
df_tmp['count'] = df_tmp['file_name']
del df_tmp['file_name']
df_tmp['rac'] = False
df_tmp.loc['2020-09-22':, 'rac'] = True  # type: ignore

fig = plt.figure(figsize=(10, 10))
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.barplot(y=df_tmp.index, x=df_tmp['count'], hue=df_tmp['rac'])
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_files_byday.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

df_tmp = pd.DataFrame(df['file_name'].resample('1H').count())
df_tmp['count'] = df_tmp['file_name']
del df_tmp['file_name']
df_tmp['rac'] = False
df_tmp.loc['2020-09-22':, 'rac'] = True  # type: ignore
df_tmp = df_tmp.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour

fig = plt.figure(figsize=(10, 10))
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.barplot(y=df_tmp['hour'], x=df_tmp['count'], hue=df_tmp['rac'], orient='h')
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_files_byhour.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

# %% [markdown]
# There are some data gaps, especially in the pre-ractopamine subset.
#
# There needs to be a resampling to rebalance classes, prioritizing samples
# that are closer to the start of the RAC provision.

# %% [markdown]
# ## Visualizing waveform and spectrogram of random sample

# %%
file_name = np.random.choice(df['file_name'])
x, sr = lr.load(file_name)

fig, ax = plt.subplots(2, 1, sharex=False, sharey=False, figsize=(16, 9))
lrdisp.waveplot(
    x,
    sr=sr,
    ax=ax[0],
)
img = lrdisp.specshow(lr.amplitude_to_db(np.abs(lr.stft(x)), ref=np.max),
                      y_axis='log',
                      x_axis='time',
                      ax=ax[1])
fig.colorbar(img, ax=ax[1], format="%+2.0f dB")
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_audio_sample.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

# %% [markdown]
# # Visualizing basic dataset properties

# %% [markdown]
# ### Shape

# %%
print("Shape:", df.shape)
print(df.loc[:, 'rac'].groupby(df.loc[:, 'rac']).count())
print(df.info())

# %% [markdown]
# Due to the high amount of extracted features, there must be a selection
# of the most relevant ones for further analysis with advanced methods
# (e.g., SOM)

# %% [markdown]
# ### Class distribution

# %%
df_melt = pd.melt(df, value_vars=['rac'], value_name='ractopamine')
fig = plt.figure(figsize=(10, 10))
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
ax = sns.countplot(data=df_melt, x='ractopamine', hue='ractopamine')

for p in ax.patches:
    ax.annotate(f'\n{p.get_height()}', (p.get_x() + 0.2, p.get_height()),
                ha='center',
                va='top',
                color='white',
                size=18)
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_rac_distribution.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

# %% [markdown]
# There's a substantial imbalance between the subsets.
# It's not an obstacle for this analysis. However, for the next steps,
# it may be desirable to perform an upsampling/downsampling on the dataset.

# %% [markdown]
# ### Zero Crossings
# The Zero crossing rate (ZCR) measures the number of times that the time
# domain signal changes its sign. Even though it is computed in the time
# domain, it describes the amount of high-frequency energy in the signal
# (i.e., 'brightness') and correlates strongly with the spectral centroid
# ZCR has also proven to be quite discriminative for classes of percussion
# instruments.

# %%
print(df.loc[:, ['zero_crossing_rate', 'zero_crossings']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['zero_crossing_rate'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    kde=True,
    rug=True,
    row='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_zero_crossing_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['zero_crossing_rate'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    row='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_zero_crossing_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# The distribution of the zero crossing seems slightly different after the
# ractopamine provision.
#
# As noted by Ricardo, this may be related to an increase in the animals'
# vocal activity.

# %% [markdown]
# ### Spectrogram and Mel-Spectrogram
# There exists only one Fourier transform of a given signal.
# However, there is an infinite number of time-frequency representations.
# The most popular one is the spectrogram, defined as the Fourier transform of
# successive signal frames. Frames are widely used in audio processing
# algorithms. They are portions of the signal with given time localizations.
#
# Standard window shapes are Gaussian, Hamming, Hanning, or rectangular and
# typical frame durations are from 20 ms to 100 ms in audio processing. As a
# rule of thumb, rectangular windows should not be used in practice, except
# under special circumstances. From frames, it is easy to build short time
# Fourier transforms (STFTs) as the FT of successive frames.
#
# Spectrograms are energy representations and they are defined as the squared
# modulus of the STFT.
#
# The mel-spectrogram is an spectrogram, converted to the Mel scale

# %%
print(df.loc[:, ['spectrogram', 'mel_spectrogram']].describe())
df_melt = pd.melt(df,
                  id_vars=['rac'],
                  value_vars=['spectrogram', 'mel_spectrogram'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectrogram_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectrogram', 'mel_spectrogram'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectrogram_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Harmonics and Perceptual Shock Wave
# Harmonics are characteristichs that represent the sound color
# Perceptrual shock wave represents the sound rhythm and emotion

# %%
print(df.loc[:, ['harmonics', 'perceptual_shock_wave']].describe())
df_melt = pd.melt(df,
                  id_vars=['rac'],
                  value_vars=['harmonics', 'perceptual_shock_wave'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_harmonics_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['harmonics', 'perceptual_shock_wave'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_harmonics_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Spectral Centroids
# In addition to the rough spectrum described by the MFCCs and bandwise
# energy descriptors, more simple spectral shape features are also useful.
# These include the first four moments of the spectrum, i.e.,
# spectral centroid, spectralspread, spectral skewness, and spectral kurtosis.
#
# The spectral centroid is the "center-of-mass" of the analysed audio file.

# %%
print(df.loc[:, ['spectral_centroids']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['spectral_centroids'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_centroids_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectral_centroids'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_centroids_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# The distribution of the spectral centroids seems to have a displacement to
# higher values, after the provision of ractopamine to the animals.
#
# This seems to suggest that the most intense peaks of audio were at higher
# frequencies, with rac=True.

# %% [markdown]
# ### Spectral Centroids (Delta and Accelerate)

# %%

print(df.loc[:, ['spectral_centroids_delta', 'spectral_centroids_accelerate']].
      describe())
df_melt = pd.melt(
    df,
    id_vars=['rac'],
    value_vars=['spectral_centroids_delta', 'spectral_centroids_accelerate'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(
    f'./output_{FILE_PREFIX}/{FILE_PREFIX}'
    '_spectral_centroids_derivatives_rac.png',
    bbox_inches='tight',
    transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(
    df_tmp,
    id_vars=['day_quarter'],
    value_vars=['spectral_centroids_delta', 'spectral_centroids_accelerate'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(
    f'./output_{FILE_PREFIX}/{FILE_PREFIX}'
    '_spectral_centroids_derivatives_day.png',
    bbox_inches='tight',
    transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Chroma features
# The chroma vector is a perceptually motivated feature vector using the
# concept of chroma in Shepard's helix representation of musical pitch
# perception. According to Shepard [584], the perception of pitch with respect
# to a musical context can be graphically represented by using a continually
# cyclic helix that has two dimensions, chroma and height. Chroma refers to the
# position of a musical pitch within an octave that corresponds to a cycle of
# the helix.

# %%

# Visualizing single-file chromagram
chroma = lr.feature.chroma_stft(y=x, sr=sr, n_chroma=12, n_fft=2048)
fig, ax = plt.subplots(figsize=(16, 9))
img = lr.display.specshow(chroma, y_axis='chroma', x_axis='time', ax=ax)
fig.colorbar(img, ax=ax)
ax.set(title='Chromagram')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_chromagram.png',
            bbox_inches='tight',
            transparent=True)

chroma_cols = [
    'chroma1',
    'chroma2',
    'chroma3',
    'chroma4',
    'chroma5',
    'chroma6',
    'chroma7',
    'chroma8',
    'chroma9',
    'chroma10',
    'chroma11',
    'chroma12',
]
print(df.loc[:, chroma_cols].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=chroma_cols)
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    row='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_chroma_features_rac.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# Chroma features may be very useful for distinguishing musical tones.
# However, related research papers didn't seem to use it for pig audio
# analysis. With this, they are discard candidates.
#
# Still, there were some interesting changes in distribution for chroma7,
# chroma8, chroma9, chroma11, and chroma12, which should be further analyzed.

# %%
plt.figure(figsize=(12, 12))
sns.heatmap(df.loc[:, chroma_cols + ['rac']].corr(),
            vmin=-1.0,
            vmax=1.0,
            cmap="coolwarm",
            center=0.0,
            robust=True,
            annot=True,
            fmt='.1f')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_chroma_features_corr.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# As expected, the chroma features are highly correlated with their neighbors.
#
# When comparing to our class (rac), the hightest absolute correlation between
# it and a chroma feature is 0.15 which is not very substantial.

# %% [markdown]
# ### Tempo (BMP)
# Estimate mean of the tempo (beats per minute), as extracted by librosa.

# %%
print(df.loc[:, ['tempo_bpm']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['tempo_bpm'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_tempo_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp, id_vars=['day_quarter'], value_vars=['tempo_bpm'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_tempo_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# This feature, similarly to the chroma features, is mostly used with music
# analysis. No relevant differences in the distributions were observed.
#
# This feature is a discard candidate.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Spectral Rolloff (85%)
# Spectral roll-off is defined as the frequency index R below which a certain
# fraction (85%) of the spectral energy resides.

# %%
print(df.loc[:, ['spectral_rolloff']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['spectral_rolloff'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_rolloff_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectral_rolloff'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_rolloff_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# A slight displacement can be seem between the two groups of files.
# This could reinforce the hypothesis drawn from the spectral centroid
# analysis, as the 85% threshold is at a higher frequency after RAC provision.

# %% [markdown]
# ### Spectral Flux
# The spectral flux, also known as the delta spectrum magnitude, is a measure
# of local spectral change. It is defined as the squared norm of the
# frame-to-frame spectral difference.

# %%
print(df.loc[:, ['spectral_flux']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['spectral_flux'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_flux_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectral_flux'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_flux_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# There has been no noticeable major changes in the distribution.
# This suggests that the number of "peaks"/onsets is similar after the rac
# provision.
# This feature is a discard candidate.

# %% [markdown]
# ### Spectral Bandwidth
# The bandwidth of the spectrum is described by spectral spread
#
# The spectral bandwidth 1 at frame t is computed by:
#
# (sum_k S[k, t] * (freq[k, t] - centroid[t])**p)**(1/p)

# %%
print(df.loc[:, [
    'spectral_bandwidth_2', 'spectral_bandwidth_3', 'spectral_bandwidth_4'
]].describe())
df_melt = pd.melt(df,
                  id_vars=['rac'],
                  value_vars=[
                      'spectral_bandwidth_2', 'spectral_bandwidth_3',
                      'spectral_bandwidth_4'
                  ])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(
    f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_bandwidths_rac.png',
    bbox_inches='tight',
    transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Correlation matrix of subset of features

# %%
plt.figure(figsize=(12, 12))
sns.heatmap(df.loc[:, [
    'zero_crossing_rate', 'spectrogram', 'mel_spectrogram', 'harmonics',
    'perceptual_shock_wave', 'spectral_centroids', 'tempo_bpm',
    'spectral_rolloff', 'spectral_flux', 'spectral_bandwidth_2',
    'spectral_bandwidth_3', 'spectral_bandwidth_4', 'rac'
]].corr(),
            vmin=-1.0,
            vmax=1.0,
            cmap="coolwarm",
            center=0.0,
            robust=True,
            annot=True,
            fmt='.1f')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_corr.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# The spectral centroids feature had the highest correlation with 'rac',
# between the analyzed features.

# %% [markdown]
# ### MFCC
# Mel-frequency cepstral coefficients (MFCCs) describe the rough shape of
# the signal spectrum and are widely used in speech recognition [536].
# Similarly, they are often encountered in percussion transcription algorithms.
# Usually the coefficients are calculated in short (about 20 ms) partially
# overlapping frames over the analysed segment, and 5 to 15 coefficients are
# retained in each frame. Instead of using these directly as features,
# typically the mean and variance of each coefficient over the segment are
# used. In addition, the first- and second-order temporal differences of the
# coefficients, and the means and variances of these, are commonly used as
# features.

# %%

# Visualizing single-file mfccs
mfccs = lr.feature.mfcc(y=x, sr=sr, n_mfcc=20)
fig, ax = plt.subplots(figsize=(16, 9))
img = lr.display.specshow(mfccs, x_axis='time', ax=ax)
fig.colorbar(img, ax=ax)
ax.set(title='MFCC')
plt.draw()
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_mfcc.png')

mfcc_cols = [
    'mfcc0',
    'mfcc1',
    'mfcc2',
    'mfcc3',
    'mfcc4',
    'mfcc5',
    'mfcc6',
    'mfcc7',
    'mfcc8',
    'mfcc9',
    'mfcc10',
    'mfcc11',
    'mfcc12',
    'mfcc13',
    'mfcc14',
    'mfcc15',
    'mfcc16',
    'mfcc17',
    'mfcc18',
    'mfcc19',
]
print(df.loc[:, mfcc_cols].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=mfcc_cols)
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    row='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_mfccs_rac.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# From related papers, the MFCCs are some of the most representative features.
# The main question related the MFCCs is the number of coefficients that will
# be used. For most use cases, 12 or 13 coefficients seem to be enough.
#
# TODO Study and write detailed description of MFCC in monograph.
#
# mfcc14 through mfcc19 are discard candidates.

# %%
plt.figure(figsize=(12, 12))
sns.heatmap(df.loc[:, mfcc_cols + ['rac']].corr(),
            vmin=-1.0,
            vmax=1.0,
            cmap="coolwarm",
            center=0.0,
            robust=True,
            annot=True,
            fmt='.1f')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_mfccs_corr.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# Initially the highest absolute correlation with 'rac' is onm mfcc9, mfcc15,
# and mfcc16.
#
# The coefficients by themselfs seem to be relatively independent of each
# other, taking into account only the correlation matrix.
# This may be an indicator of high representativity of the dataset.

# %% [markdown]
# ### Visualizing discard candidates

# %%
discard_features = [
    'zero_crossings',
    'spectrogram',
    'mel_spectrogram',
    'harmonics',
    'perceptual_shock_wave',
    'tempo_bpm',
    'spectral_flux',
    'spectral_bandwidth_2',
    'spectral_bandwidth_3',
    'spectral_bandwidth_4',
] + chroma_cols + [
    'mfcc14',
    'mfcc15',
    'mfcc16',
    'mfcc17',
    'mfcc18',
    'mfcc19',
    'mfcc14_delta',
    'mfcc15_delta',
    'mfcc16_delta',
    'mfcc17_delta',
    'mfcc18_delta',
    'mfcc19_delta',
    'mfcc14_accelerate',
    'mfcc15_accelerate',
    'mfcc16_accelerate',
    'mfcc17_accelerate',
    'mfcc18_accelerate',
    'mfcc19_accelerate',
]

print('# of discarded features:', len(discard_features))
print(
    '# of remaining features:',
    df.loc[:, 'zero_crossing_rate':].shape[1] -  # type: ignore
    len(discard_features))

# %% [markdown]
# # Final comments
# After analyzing the extracted features for our dataset, the following
# features are discard candidates:
#
# - spectrogram';
# - mel-spectrogram';
# - harmonics';
# - perceptual shock wave;
# - chroma features;
# - tempo (bpm);
# - spectral flux;
# - spectral bandwidth;
# - mfcc14, ..., mfcc19.
#
# For now, the remaining features shall be used for further analysis and
# data visualization methods, such as SOMs.
#
# If necessary, more features, such as deltas and accelerates, can be
# discarded, for performance reasons. The relevance of these features should
# be further analyzed.

# %%


from pathlib import Path

import mlflow
import tensorflow as tf
import yaml
from loguru import logger
from tensorflow.keras.models import load_model

from utils import get_sorted_runs

with open("configs/params.yaml") as reproducibility_params:
    mlflow_config = yaml.safe_load(reproducibility_params)["mlflow"]

experiment_name = mlflow_config["experiment_name"]

def load_model_artifact() -> tf.keras.Model:
    """Load artifacts for a particular `run_id`.
    Args:
        run_id (str): ID of the run to load model artifacts from.
    Returns:
        Artifacts needed for inference.
    """

    all_runs = get_sorted_runs(
        experiment_name=experiment_name,
        order_by=["metrics.val_loss ASC"],
    )

    print(
        all_runs[
            [
                "run_id",
                "tags.mlflow.runName",
                "metrics.val_categorical_accuracy",
                "metrics.val_loss",
            ]
        ],
    )

    best_run = all_runs.iloc[0]["run_id"]

    logger.info(f"Best run id is : {best_run}")

    # Load model
    run = mlflow.get_run(run_id=best_run)
    homedir = Path(run.info.artifact_uri).parent.parent.parent.parent

    root = Path(run.info.artifact_uri).relative_to(homedir)
    model_url = Path(root) / Path("model/data") / "model.h5"

    model = load_model(model_url)
    logger.info(f"Model loaded from {run.info}")

    return model

if __name__ == "__main__":
    load_model_artifact()


from __future__ import print_function
import math

TAX = 0.08875
GRATUITY = 0.25

def you_owe(menu, meal):
    food = 0
    for m in meal:
        if m in menu:
            food += meal[m] * menu[m]
    tax = food * TAX
    gratuity = (food + tax) * GRATUITY
    total = food + tax + gratuity
    return math.floor(100 * total) / 100

if __name__ == '__main__':
    menu1 = {"calamari":5, "carbonara":15, "branzino":15, "cheesecake":7.50}
    meal1 = {"calamari":1, "carbonara":1, "branzino":1, "cheesecake":2}
    menu2 = {"a": 4.5, "b": 3.4, "c": 9}
    meal2 = {"a": 1, "b": 5}
    print(you_owe(menu1, meal1))
    print(you_owe(menu2, meal2))


from class_def import Rectangle, Square, Circle

r1 = Rectangle(2, 4)
r2 = Rectangle(60, 80)

# print(r1.get_rect_area())
# print(r2.get_rect_area())

s1 = Square(8)
s2 = Square(0.8)

c1 = Circle(8)
c2 = Circle(0.8)

list_area = [r1, r2, s1, s2, c1, c2 ]
# Without Polymorphysm:
# for i in list_area:
#     if isinstance(i, Rectangle):
#         print (i.get_rect_area())
#     elif isinstance(i, Square):
#         print(i.get_sqare_area())
#     elif isinstance(i, Circle):
#         print(i.get_cir_area())

# Using Polymorphysm:
for i in list_area:
    print (i.get_area())

for i in list_area:
    print (i)


# https://atcoder.jp/contests/abc095/tasks/arc096_a
# brute-force, computational complexity O(max(X, Y))

A, B, C, X, Y = map(int, input().split())

INF = float('inf')
min_cost = INF

k = 0
while X - k >= 0 or Y - k >= 0:
    cost = A * max(0, X-k) + B * max(0, Y-k) + 2 * C * k
    min_cost = min(min_cost, cost)
    k += 1

print(min_cost)


import time, datetime

t1 = '2015-07-31T18:57:04Z'
t2 = '2015-08-01T02:34:00Z'

st1 = time.strptime(t1, '%Y-%m-%dT%H:%M:%SZ')   # Convert to the time struct
st2 = time.strptime(t2, '%Y-%m-%dT%H:%M:%SZ')

dt = time.mktime(st2) - time.mktime(st1)        # Convert to seconds
dt_str = str(datetime.timedelta(seconds=dt))    # Convert to the format as HH:MM:SS

print(dt_str)

#
# Add a timedelta using time.strptime
#
t3 = '2013070100'
st3 = time.strptime(t3, '%Y%m%d%H')
dt = time.mktime(st3) + 30*3600     # 30 hours (seconds)
sdt = time.localtime(dt)
print(time.strftime("%Y%m%d%H", sdt))

#
# Add a timedelta using datetime.datetime.strptime
#
st4 = '2015070100'
t4 = datetime.datetime.strptime(st4, '%Y%m%d%H')
print('t0     = {}'.format(t4))
for hour in [-3, -2, -1, 0, 1, 2, 3]:
    t5 = t4 + datetime.timedelta(hours=hour)
    st5 = t5.strftime('%Y%m%d%H')
    sign = '-' if hour < 0 else '+'
    print('t0 {} {} = {}'.format(sign, abs(hour), st5))


self.description = "-S provision"

sp = pmpkg("pkg1")
sp.provides = ["provision=1.0-1"]
self.addpkg2db("sync", sp)

self.args = "-S provision"

self.addrule("PACMAN_RETCODE=0")
self.addrule("PKG_EXIST=pkg1")


# -*- coding: utf-8 -*-
"""
Created on Mon Dec  3 15:09:47 2018

@author: HP
"""

a=input("")
b=input("")
c=input("")

def fanhui(a,b,c):
    #得到请求网址后返回的字符串
    from urllib.request import urlopen
    wangzhi=('http://cs1110.cs.cornell.edu/2016fa/a1server.php?from='\
             +a+"&to="+b+"&amt="+c)
    doc = urlopen(wangzhi)
    docstr = doc.read()
    doc.close()
    jstr = docstr.decode('ascii')
    jstr=jstr.split()
    return(jstr)

def tiqu(jstr):
    #在返回的字符串中提取出转换后值
    qian=[]
    for i in jstr:
        al=len(i)
        shu=str("0123456789")
        ji=0
        for b in range(al):
            zhao=i[b]
            if shu.count(zhao)==1:
                ji=ji+1
            else:
                ji=ji
        if ji!=0:
            i=[i]
            qian=qian+i
        #现在就只有转换前和转换后两个数了
    if len(qian)==2:
        qian=qian
    else:
        #间接判断输入是否合法
        print("错误！")
        return None
    qian=qian[1]
    mubiao=qian[1:]
    return(mubiao)

def exchange(currency_from, currency_to, amount_from):
    a=fanhui(currency_from, currency_to, amount_from)
    b=tiqu(a)
    return(b)

d=exchange(a,b,c)
print(d)

def test_fanhui():
    #测试返回函数
    jstr=fanhui("USD","EUR","2.5")
    assert jstr==['{', '"from"', ':', '"2.5', 'United', 'States',\
                    'Dollars",', '"to"', ':', '"2.1589225', 'Euros",',\
                    '"success"', ':', 'true,', '"error"', ':', '""', '}']
    jstr=fanhui("USD","AED","2.5")
    assert jstr==['{', '"from"', ':', '"2.5', 'United', 'States',\
                    'Dollars",', '"to"', ':', '"9.182905', 'United', \
                    'Arab', 'Emirates', 'Dirhams",', '"success"', ':',\
                    'true,', '"error"', ':', '""', '}']
    jstr=fanhui("CAD","MMK","3")
    assert jstr==['{', '"from"', ':', '"3', 'Canadian', 'Dollars",', \
                    '"to"', ':', '"3508.824587556', 'Myanma', 'Kyats",',\
                    '"success"', ':', 'true,', '"error"', ':', '""', '}']

def test_tiqu():
    #测试提取函数
    b=tiqu(['{', '"from"', ':', '"2.5', 'United', 'States',\
                    'Dollars",', '"to"', ':', '"2.1589225', 'Euros",',\
                    '"success"', ':', 'true,', '"error"', ':', '""', '}'])
    assert b=="2.1589225"
    b=tiqu(['{', '"from"', ':', '"2.5', 'United', 'States',\
                    'Dollars",', '"to"', ':', '"9.182905', 'United', \
                    'Arab', 'Emirates', 'Dirhams",', '"success"', ':',\
                    'true,', '"error"', ':', '""', '}'])
    assert b=="9.182905"
    c=tiqu(['{', '"from"', ':', '"3', 'Canadian', 'Dollars",', \
                    '"to"', ':', '"3508.824587556', 'Myanma', 'Kyats",',\
                    '"success"', ':', 'true,', '"error"', ':', '""', '}'])
    assert c=="3508.824587556"

def test_exchange():
    #测试exchange函数
    d=exchange("USD","EUR","2.5")
    assert d=="2.1589225"
    d=exchange("USD","AED","2.5")
    assert d=="9.182905"
    d=exchange("CAD","MMK","3")
    assert d=="3508.824587556"

def testAll():
    #测试全部
    test_fanhui()
    test_tiqu()
    test_exchange()
    print("All tests passed!")

testAll()


from transformers import pipeline
import torch
import pandas as pd
import time
from csv import DictWriter
device = 0 if torch.cuda.is_available() else -1
field_names = ['metin','label','kategori','multi_class_true','multi_class_false']

data = {
    'metin':[],
    'label':[],
    'kategori':[],
    'multi_class_true':[],
    'multi_class_false':[]
}
df = pd.DataFrame(data)
df.to_csv('Result_Zero_Shot.csv', index = False)

classifier = pipeline("zero-shot-classification", model="joeddav/xlm-roberta-large-xnli", device=device)
df = pd.read_csv('your_data_csv_format')
labels = df['kategori'].unique()

for i in range(len(df['metin'])):
    input_text1 = df['metin'][i]
    multi_true_result = classifier(input_text1, labels,multi_class=True)
    multi_false_result = classifier(input_text1, labels,multi_class=False)
    row_dict = {
        'metin':df['metin'][i], 'label':df['label'][i], 'kategori':df['kategori'][i], 'multi_class_true':multi_true_result ,'multi_class_false':multi_false_result
        }

    with open('result.csv', 'a+', newline='') as write_obj:
        dict_writer = DictWriter(write_obj, fieldnames=field_names)
        dict_writer.writerow(row_dict)
    #print(result)


#!/usr/bin/python

from optparse import OptionParser
from collections import defaultdict
import os,sys,warnings,re,glob
import logging
import subprocess
import TrieSearch

def main():
	usage = 'usage: %prog <options> -p junctionPep -c percolatorfile -t tmpdir -b bedtooldir -o outfile'
	parser = OptionParser(usage)
	parser.add_option('-p', dest='pepfile', help='junction pep file [Default %default]')
	parser.add_option('-c', dest='cruxfile', help='percolator.target.peptides.txt from crux [Default %default]')
	parser.add_option('-t', dest='tmpdir',default='tmp',help='tmp dir [Default %default]')
	parser.add_option('-b', dest='bedtooldir',help='bedtool bin directory path')
	parser.add_option('-o', dest='outfile',default='result.txt',help='final output file')

	(options, args) = parser.parse_args()

	if options.pepfile is None or options.cruxfile is None or options.bedtooldir is None:
		sys.exit("[ERROR] "+parser.get_usage())

	warnings.formatwarning = custom_formatwarning

	## start
	# step 1: '''read percolator peptides under FDR 0.05'''
	sample = os.path.basename(options.pepfile)
	warnings.warn('======'+sample)
	warnings.warn('# Obtain percolator result:\t' + options.cruxfile)

	pephash,firstAA = {},{}
	with open(options.cruxfile,'r') as f:
		for line in f:
			ele = line.rstrip().split("\t")
			if ele[0] != 'PSMId' and float(ele[2]) < 0.05:
				pep = re.sub(re.compile("\.|\*|-"),'',ele[4])
				pephash[pep] = ele[3]
				firstAA[str.upper(pep)[0]] = ''

	# ste 2: ''' store junction pep into dict'''
	warnings.warn("# Read Junction Pep fasta:\t" + options.pepfile)
	seq = defaultdict(str)
	head = ''
	with open(options.pepfile,'r') as f:
		for line in f:
			if line[0] == '>':
				head = line.rstrip()[1:]
			else: seq[head] += line.rstrip()

	# step 3: ''' Re-map percolator peptides to Junction Pep'''
	warnings.warn('# Re-map percolator peptides to Junction Pep')
	info = search(pephash,seq,firstAA)

	# step 4: ''' FDR filter'''
	warnings.warn('# Get peptide under fdr')
	allPepHash = defaultdict(dict)
	novelChrPepHash = {}
	for peptide in info:
		ks = info[peptide]
		tag = 0
		for k in ks:
			head = k.split("\t")[1]
			if head.find('startTrim:') == -1: # if peptide could be mapped to uniprot, means annotated peptide
				tag = 1
		if tag == 0:
			novelChrPepHash[peptide] = pephash[peptide]

	warnings.warn("# Num of pep from novel junction:\t %s" % len(novelChrPepHash))
	warnings.warn("# Num of pep from annot junction:\t %s" % (len(pephash) - len(novelChrPepHash)))

	result = ['# Peptide\tProtein\tPEP\tKnown/Novel']
	for peptide in pephash.keys():
		if peptide in novelChrPepHash: continue
		ks = info[peptide]
		for k in ks:
			s = k+"\t"+pephash[peptide] + "\t1"
			result.append(s)

	localResult = localFDR(novelChrPepHash)
	arr = re.split(re.compile(";"),localResult)
	for a in arr:
		ele = a.split("\t")
		ks = info[ele[0]]
		for k in ks:
			s = k + "\t"+pephash[ele[0]] + "\t0"
			result.append(s)

	# step 5: ''' output to files'''
	with open(options.outfile,'w') as fout:
		fout.write('======'+sample+'\n')
		fout.write('# Obtain percolator result from ' + options.cruxfile + '\n')
		fout.write('# Read Junction Pep fasta:\t' + options.pepfile + '\n')
		for r in result:
			fout.write(r+'\n')

	## end
def localFDR(chrPepHash,FDR=0.01):
	if len(chrPepHash) == 0: return ''
	pepvalues = [float(x) for x in chrPepHash.values()]
	uniq = sorted(list(set(pepvalues)))
	warnings.warn("# Before FDR %f filter:\t%d" %(FDR,len(chrPepHash)))
	result,n = '',0
	for s in uniq:
		sums,content,num,total = 0.0,'',0,len(chrPepHash.keys())
		for l in chrPepHash:
			pep = float(chrPepHash[l])
			if pep <= s:
				sums += pep
				num += 1
				content += str(l)+"\t"+str(pep)+";"
		fdr = sums / num
		if fdr > FDR: break
		n += 1
		result = content
	warnings.warn( "# After FDR %f filter:\t%d" %(FDR,n))
	return result

def search(pephash,seqs,firstAA):
	info = defaultdict(dict)
	trieTree = TrieSearch.TrieSearch(pephash.keys())
	trieTree.make_trie()
	for aa in firstAA:
		warnings.warn("## AA:\t"+aa)
		result = trieTree.doSearch(seqs,[aa])
		for r in result:
			ele = r.split('\t')
			pep,ids = ele[1],ele[2].split(';')
			for i in ids:
				info[pep]['\t'.join([pep,i])] = ''
	return info

def custom_formatwarning(msg, *a):
	# ignore everything except the message
	return str(msg) + '\n'

if __name__ == '__main__':
	main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import gensim
import os
import codecs
import pandas as pd
from sklearn import linear_model
import numpy as np
import StringIO

model_en = gensim.models.KeyedVectors.load_word2vec_format(os.path.expanduser("~/wiki.en.vec"))
model_ta = gensim.models.KeyedVectors.load_word2vec_format(os.path.expanduser("~/wiki.ta.vec"))

with open("top_en_ta.csv", "rb") as myfile:
    decoded = codecs.decode(myfile.read(), "utf-8", "ignore")
    encoded = codecs.encode(decoded, "utf-8", "ignore")
    fakefile = StringIO.StringIO(encoded)

df = pd.read_csv(fakefile, encoding='utf-8')
for n in range(len(df)):
    if df['tamil'][n] not in model_ta.vocab or df['english'][n] not in model_en.vocab:
        df = df.drop(n)
df = df.reset_index(drop=True)

df['vector_ta'] = [model_ta[df['tamil'][n]] for n in range(len(df))]
df['vector_en'] = [model_en[df['english'][n]] for n in range(len(df))]
ma_train_ta = pd.DataFrame(df['vector_ta'][:6000].tolist()).values
ma_train_en = pd.DataFrame(df['vector_en'][:6000].tolist()).values

def most_similar_vect(self, vect_enter, top_n=10):
    self.init_sims()
    vect_unit = gensim.matutils.unitvec(vect_enter)
    dists = np.dot(np.squeeze(np.asarray(self.syn0norm)), np.squeeze(np.asarray(vect_unit)))
    if not top_n:
        return dists
    best = np.argsort(dists)[::-1][:top_n]
    # ignore (don't return) words from the input
    result = [(self.index2word[sim], float(dists[sim])) for sim in best]
    return result[:top_n]

"""
ta to en
"""
clf_ta_en = linear_model.LinearRegression()
clf_ta_en.fit(ma_train_ta, ma_train_en)

def traducwithsco_ta_to_en(w, numb=10):
    return most_similar_vect(model_en, clf_ta_en.predict(model_ta[w]), numb)

def traduclist_ta_to_en(w, numb=10):
    return [traducwithsco_ta_to_en(w, numb)[k][0] for k in range(numb)]

for n in range(6000, 6010):
    print(df['english'][n])
    print(df['tamil'][n])
    print(traduclist_ta_to_en(df['tamil'][n], 5))

"""
clearing
தீர்வு
[u'solve', u'solution', u'solving', u'resolving', u'solutions']

mess
குழப்பம்
[u'confusion', u'misunderstandable', u'misinterpration', u'confusing', u'misinterpretting']

reward
வெகுமதி
[u'exorbitantly', u'exorbitant', u'profiteering', u'enticements', u'disincentivise']

diet
உணவில்
[u'nutritionally', u'antinutritional', u'uncooked', u'nutritive', u'food']

rounds
சுற்று
[u'circuit', u'circuit,', u'circuit\u2014the', u'circuits', u'circuitz']

frustration
ஏமாற்றம்
[u'disappointed', u'faltering', u'embarrassment', u'desperation', u'embarrasment']

treaty
ஒப்பந்தம்
[u'renegotiated', u'renegotiation', u'agreement', u'renegotiations', u'renegotiate']

planted
நடப்படுகிறது
[u'happens', u'fluctuates', u'takes', u'occurs', u'puts']

degradation
சீரழிவு
[u'ameliorating', u'deterioration', u'ameliorative', u'amelioration', u'ameliorations']
"""

traductest_ta_to_en = [df['english'][n] in traduclist_ta_to_en(df['tamil'][n], 1) for n in range(6000, 6500)]
scorefinal_ta_to_en = sum(traductest_ta_to_en) / len(traductest_ta_to_en)
print(scorefinal_ta_to_en)

"""
en to ta
"""
clf_en_ta = linear_model.LinearRegression()
clf_en_ta.fit(ma_train_en, ma_train_ta)

def traducwithsco_en_to_ta(w, numb=10):
    return most_similar_vect(model_ta, clf_en_ta.predict(model_en[w]), numb)

def traduclist_en_to_ta(w, numb=10):
    return [traducwithsco_en_to_ta(w, numb)[k][0] for k in range(numb)]

for n in range(7000, 7010):
    print(df['tamil'][n])
    print(df['english'][n])
    for i in traduclist_en_to_ta(df['english'][n], 5):
        print(i)

"""
கலைப்பு
liquidation
நஷ்டஈடு
கடனீட்டு
கடனாளி
கடனளிப்பு
ஏஐஜி

கவரும்
lure
முயலுகையில்
விரட்ட
மீன்பிடிக்க
துரத்திக்கொண்டு
காப்பாற்றிக்கொள்ள

விசைப்பலகைகள்
keyboards
pcகள்
pdaகள்
cpuகள்
ஏபிஐகள்
lcdகள்

தீர்க்கரேகை
meridian
ஐகன்சு–பிரனெல்
அட்சரேகை
அதிபரவளைவின்
காம்பர்ட்சேவ்

தூக்க
sleeping
குளியலறையில்
வயிறார
கழிப்பறையில்
தூங்கும்
தூங்காத

தக்கவைத்து
retaining
தக்கவைத்திருக்கும்
hpfs
தக்கவைத்துக்கொள்வதற்காக
தக்கவைக்கும்
மாற்றியமைப்பதற்கான

இடைவெளி
interval
μl
−∞
v/r
λm
a,bஇல்

சுதந்திரம்
liberty
எட்ஜ்பாஸ்டன்
கம்னியூஸ்ட்
ரீகன்புக்ஸ்
கம்னியூஸ்டு
லேக்ஹவுஸ்

பற்கள்
teeth
நகங்கள்
விரலெலும்புகள்
இடுப்பெலும்புகள்
களிம்புகள்
முலைக்காம்புகள்

வாட்கின்ஸ்
watkins
ஹோர்டன்
டேவ்சன்
ஜெஃப்சன்
டாம்லின்சன்
ரொபின்சன்
"""


"""
单变量 x0 = 1
      alfa = .2
      梯度下降算法
"""
def gradient_descent(x0, num_iterations=100, alfa=.3):
    tmp = x0
    for num in range(num_iterations):
        tmp = tmp - alfa * 2 * tmp

    return tmp

tmp = gradient_descent(1005, 25, .4)
print('%.25f' %tmp)


import sys
import json
import pandas as pd
import yfinance as yf
import pandas_ta as ta
from backtestEngine import Strategy, Engine
import _lib

strategy_dandd = []

def read_json_file(file_path):
    with open(file_path, 'r') as f:
        return json.load(f)

def write_json_file(file_path, data):
    with open(file_path, 'w') as f:
        json.dump(data, f)

def compare_operations(a, b, operation, data, idx):
    try:
        val_a = data.loc[idx, a]
        val_b = data.loc[idx, b]
    except KeyError as e:
        print(f"KeyError: {e}")
        print("Columns in data:")
        print(data.columns)
        print(f"a = {a}")
        print(f"b = {b}")
        return False

    operations = {
        'gt': val_a > val_b,
        'lt': val_a < val_b,
        'gte': val_a >= val_b,
        'lte': val_a <= val_b,
        'eq': val_a == val_b,
        'ne': val_a != val_b,
        'ca': _lib.crossover(data, a, b, idx),
        'cb': _lib.crossbelow(data, a, b, idx),
    }
    return operations.get(operation, False)

class SMACrossover(Strategy):
    def on_bar(self):
        if self.current_idx != 0:
            for index, i in enumerate(strategy_dandd):
                if 'SubFilter' in i:
                    if compare_operations(i['A'], i['B'], i['Operation'], self.data, self.current_idx) and compare_operations(i['subFilter']['A'], i['B'], i['subFilter']['Operation'], self.data, self.current_idx):
                        self.process_strategy(i)
                else:
                    if compare_operations(i['A'], i['B'], i['Operation'], self.data, self.current_idx):
                        self.process_strategy(i)

    def process_strategy(self, strategy):
        if self.position_size == 0:
            if strategy['Action'] == 'Long_Entry':
                order_size = self.cash / self.close
                self.buy_limit('AAPL', size=order_size, limit_price=self.close)
        elif strategy['Action'] == 'Long_Exit':
            if compare_operations(strategy['A'], strategy['B'], strategy['Operation'], self.data, self.current_idx):
                self.sell_limit('AAPL', size=self.position_size, limit_price=self.close)

def create_indicator(data, indicator, attr, close, variables):
    return getattr(ta, attr)(data[close], int(variables))

def main():
    input_file_path = sys.argv[1]
    output_file_path = sys.argv[2]

    input_data = read_json_file(input_file_path)
    jsondata = input_data
    data = yf.Ticker('AAPL').history(start='2012-12-01', end='2022-12-31', interval='1d')
    e = Engine(initial_cash=100000)

    process_json_data(data, jsondata)
    e.add_data(data)
    e.add_strategy(SMACrossover())
    stats = e.run()
    print(e.portfolio_bh)
    result_data = prepare_result_data(stats)
    write_json_file(output_file_path, {"result": result_data})

def process_json_data(data, jsondata):

    lastFilterIndex = 0

    for indx, i in enumerate(jsondata):

        strategy = {}
        subStrategy = {}

        if (i['type'] == 'filter'):
            lastFilterIndex = indx
            strategy['Operation'] = i['Operation']
            strategy['Action'] = i['Action']
            if (i['Operand']['OperandType'] == 'indicator'):
                indicator = i['Operand']['Indicators']['Attribute']+"_INDX"+str(indx)+str(i['Operand']['Indicators']['Variables'][0])+i['Operand']['Indicators']['Timeframe']
                data[indicator] = getattr(ta, i['Operand']['Indicators']['Attribute'])(data['Close'], int(i['Operand']['Indicators']['Variables'][0]))
                strategy['A'] = indicator
            if (i['OperandB']['OperandType'] == 'indicator'):
                indicator = i['OperandB']['Indicators']['Attribute']+"_INDX"+str(indx)+str(i['OperandB']['Indicators']['Variables'][0])+i['OperandB']['Indicators']['Timeframe']
                # data = resample(data, i['OperandB']['Indicators']['Timeframe'], indicator, i['Operand']['Indicators']['Attribute'], 'Close', int(i['Operand']['Indicators']['Variables'][0]))
                data[indicator] = getattr(ta, i['OperandB']['Indicators']['Attribute'])(data['Close'], int(i['OperandB']['Indicators']['Variables'][0]))
                print(i['OperandB']['Indicators']['Attribute']+"_INDX"+str(indx)+str(i['OperandB']['Indicators']['Variables'][0])+i['OperandB']['Indicators']['Timeframe'])
                strategy['B'] = indicator

            if (i['Operand']['OperandType'] == 'StockAttribute'):
                indicator = (i['Operand']['StockAttribute']['Indicators'])
                print(indicator)
                strategy['A'] = indicator

            if (i['OperandB']['OperandType'] == 'StockAttribute'):
                indicator = (i['OperandB']['StockAttribute']['Indicators'])
                print(indicator)
                strategy['B'] = indicator

            if (i['Operand']['OperandType'] == 'Constant'):
                indicator = (str(i['Operand']['Constant'])+"_INDX"+str(indx))
                strategy['A'] = indicator
                data[indicator] = i['Operand']['Constant']

            if (i['OperandB']['OperandType'] == 'Constant'):
                indicator = (str(i['OperandB']['Constant'])+"_INDX"+str(indx))
                strategy['B'] = indicator
                data[indicator] = i['OperandB']['Constant']
            strategy_dandd.append(strategy)

        elif (i['type'] == 'sub-filter'):

            for jindx, j in enumerate(jsondata):

                subStrategy['Operation'] = j['Operation']

                if (j['Operand']['OperandType'] == 'indicator'):
                    indicator = j['Operand']['Indicators']['Attribute']+"_INDX"+str(jindx)+str(j['Operand']['Indicators']['Variables'][0])+j['Operand']['Indicators']['Timeframe']
                    data[indicator] = getattr(ta, j['Operand']['Indicators']['Attribute'])(data['Close'], int(j['Operand']['Indicators']['Variables'][0]))
                    subStrategy['A'] = indicator
                if (j['OperandB']['OperandType'] == 'indicator'):
                    indicator = j['OperandB']['Indicators']['Attribute']+"_INDX"+str(jindx)+str(j['OperandB']['Indicators']['Variables'][0])+j['OperandB']['Indicators']['Timeframe']
                    # data = resample(data, i['OperandB']['Indicators']['Timeframe'], indicator, i['Operand']['Indicators']['Attribute'], 'Close', int(i['Operand']['Indicators']['Variables'][0]))
                    data[indicator] = getattr(ta, j['OperandB']['Indicators']['Attribute'])(data['Close'], int(i['OperandB']['Indicators']['Variables'][0]))
                    print(j['OperandB']['Indicators']['Attribute']+"_INDX"+str(jindx)+str(j['OperandB']['Indicators']['Variables'][0])+j['OperandB']['Indicators']['Timeframe'])
                    subStrategy['B'] = indicator

                if (j['Operand']['OperandType'] == 'StockAttribute'):
                    indicator = (j['Operand']['StockAttribute']['Indicators'])
                    print(indicator)
                    subStrategy['A'] = indicator

                if (j['OperandB']['OperandType'] == 'StockAttribute'):
                    indicator = (j['OperandB']['StockAttribute']['Indicators'])
                    print(indicator)
                    subStrategy['B'] = indicator

                if (i['Operand']['OperandType'] == 'Constant'):
                    indicator = (str(j['Operand']['Constant'])+"_INDX"+str(jindx))
                    subStrategy['A'] = indicator
                    data[indicator] = j['Operand']['Constant']

                if (i['OperandB']['OperandType'] == 'Constant'):
                    indicator = (str(j['OperandB']['Constant'])+"_INDX"+str(jindx))
                    subStrategy['B'] = indicator
                    data[indicator] = j['OperandB']['Constant']

                strategy_dandd[lastFilterIndex]['subFilter'] = subStrategy

    return strategy

def prepare_result_data(stats):
    print(stats)

if __name__ == "__main__":
    main()


import torch
import torchvision
import matplotlib.pyplot as plt
from torchvision import transforms
from torch.utils.data import DataLoader
from tqdm import tqdm
from time import time
from datetime import timedelta

class dotdict(dict):
    """dot.notation access to dictionary attributes"""
    __getattr__ = dict.get
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

def load_transformed_dataset(img_size):
    data_transforms = [
        transforms.Resize((img_size, img_size)), # height, width
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(), # Scales data into [0,1]
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), # Scale between [-1, 1]
    ]
    data_transform = transforms.Compose(data_transforms)

    train = torchvision.datasets.StanfordCars(root="./", download=False,
                                         transform=data_transform)

    test = torchvision.datasets.StanfordCars(root="./", download=False,
                                         transform=data_transform, split='test')
    return train, test

def plot_images(images):
    plt.figure(figsize=(32, 32))
    plt.imshow(torch.cat([
        torch.cat([i for i in images.cpu()], dim=-1),
    ], dim=-2).permute(1, 2, 0).cpu())
    plt.show()

def make_dataloader(img_size, batch_size):

    data_train, data_test = load_transformed_dataset(img_size)
    dataset =  torch.utils.data.ConcatDataset([data_train, data_test])
    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

    return dataloader

def train_one_epoch(args, model, diffusion, dataloader, optimizer, criterion):
    if args.print_pbar: pbar = tqdm(dataloader)
    else: pbar = dataloader
    total_loss = 0
    for images, _ in pbar:
        images = images.to(args.device)
        t = diffusion.sample_timesteps(images.shape[0]).to(args.device)
        x_t, noise = diffusion.noise_images(images, t)
        predicted_noise = model(x_t, t)
        loss = criterion(noise, predicted_noise)
        # noise를 없앤 image 자체를 학습하고 싶으면 x_t와 predicted_noise를 loss의 input으로 준다.

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if args.print_pbar: pbar.set_postfix(MSE=f'{loss.item():.3f}')
        total_loss += loss.item()

    avg_loss = total_loss / len(dataloader)
    return avg_loss

def train(args, model, diffusion, optimizer, criterion, dataloader):
    print('Start Training')
    model.train()
    for epoch in range(args.epochs):
        start = time()
        avg_loss = train_one_epoch(args, model, diffusion, dataloader, optimizer, criterion)
        print(f'Epoch: [{epoch:03d}/{args.epochs}] \t loss={avg_loss:.4f} \t Took: {str(timedelta(seconds=int(time() - start)))}')
        torch.save(model, args.save_model_name + '.pt')
    print('Finish!')


import sys
from collections import deque
from itertools import permutations
input = sys.stdin.readline

dy = [-1, 0, 0, 1]
dx = [0, 1, -1, 0]

def bfs(start):
    q = deque()
    v = [[False] * w for _ in range(h)]
    res = [0] * len(dirts)
    found = 1

    q.append((start[0], start[1], 0))
    v[start[0]][start[1]] = True
    res[didx[start]] = 0

    while q:
        cy, cx, cnt = q.popleft()

        for i in range(4):
            ny = cy + dy[i]
            nx = cx + dx[i]

            if 0 <= ny < h and 0 <= nx < w:
                if board[ny][nx] != 'x' and not v[ny][nx]:

                    if (ny, nx) in didx:
                        res[didx[(ny, nx)]] = cnt + 1
                        found += 1

                    v[ny][nx] = True
                    q.append((ny, nx, cnt + 1))

    if found != len(dirts):
        return False
    return res

while True:
    w, h = map(int, input().split())
    if (w, h) == (0, 0):
        exit()

    board = []
    dirts = []
    didx = {}
    dist = []

    for i in range(h):
        tmp = list(input().rstrip())
        for j in range(w):
            if tmp[j] == 'o':
                tmp[j] = '.'
                dirts.insert(0, (i, j))

            elif tmp[j] == '*':
                dirts.append((i, j))
        board.append(tmp)

    idx = 0
    for dirt in dirts:
        didx[dirt] = idx
        idx += 1

    flag = 0
    for i in range(len(dirts)):
        t = bfs(dirts[i])
        if t:
            dist.append(t)
        else:
            flag = 1
            break
    # print(dist)

    if flag:
        print(-1)
        continue

    dirts.remove(dirts[0])
    mindist = float('inf')
    for case in permutations([i+1 for i in range(len(dirts))], len(dirts)):
        tmp = dist[0][case[0]]
        for i in range(len(case) - 1):
            tmp += dist[case[i]][case[i+1]]

        mindist = min(mindist, tmp)

    print(mindist)


from pathlib import Path
from typing import List

from pil_utils import BuildImage

from meme_generator import MemeArgsModel, add_meme
from meme_generator.exception import TextOverLength
from meme_generator.utils import make_jpg_or_gif

img_dir = Path(__file__).parent / "images"

def together(images: List[BuildImage], texts: List[str], args: MemeArgsModel):
    frame = BuildImage.open(img_dir / "0.png")
    name = args.user_infos[0].name if args.user_infos else ""
    text = texts[0] if texts else f"一起玩{name}吧！"
    try:
        frame.draw_text(
            (10, 140, 190, 190),
            text,
            weight="bold",
            max_fontsize=50,
            min_fontsize=10,
            allow_wrap=True,
        )
    except ValueError:
        raise TextOverLength(text)

    def make(img: BuildImage) -> BuildImage:
        img = img.convert("RGBA").resize((63, 63), keep_ratio=True)
        return frame.copy().paste(img, (132, 36), alpha=True)

    return make_jpg_or_gif(images[0], make)

add_meme(
    "together",
    together,
    min_images=1,
    max_images=1,
    min_texts=0,
    max_texts=1,
    keywords=["一起"],
)


from django.shortcuts import render
from django.http import HttpResponse
from django.views import View
from .articles_scrape import get_articles
from .models import Article

class Articles(View):
    def get(self, requests, *args, **kwargs):
        queryset = Article.objects.all()
        context = {
            "object_list": queryset,
            "title": "Articles"
        }
        return render(requests, "articles/articles.html", context)

class ScrapeArticles(View):
    def get(self, requests, *args, **kwargs):
        posts = get_articles()
        for i in range(len(posts)):
            article = Article()
            # print("Article: ", i + 1)
            try:
                article.title = posts[i].a.section.h3.text
                print(article.title)
            except:
                article.title = ""
                print(article.title)
            try:
                article.link = posts[i].a['href']
                print(article.link)
            except:
                article.link = ""
                print(article.link)
            try:
                article.image = posts[i].a.section.figure.img['src']
                print(article.image)
            except:
                article.image = ""
                print(article.image)
            article.save()
        context = {}
        return render(requests, "articles/scrape_articles.html", context)


import time, sys, os

# Disable
def blockPrint():
    sys.stdout = open(os.devnull, 'w')

# Restore
def enablePrint():
    sys.stdout = sys.__stdout__

def fib(f, n):
    t = time.time()
    print('n:', n)
    print()
    print(f(n))
    # print('Runtime (sec):')
    # return(time.time() - t)

def comp_fib(n):
    blockPrint()
    t0, t1 = fib(fast, n), fib(slow, n)
    enablePrint()
    print('Fast fib is {:1.2f}x faster!'.format(t1/t0))

dic = {0:0, 1:1}
def fast(n):
    if n in dic:
        return dic[n]
    else:
        dic[n] = fast(n - 2) + fast(n - 1)
        return dic[n]

def slow(n):
    if n == 0:
        return 0
    elif n == 1:
        return 1
    else:
        return slow(n - 2) + slow(n - 1)

n = 1000
while True:
    fib(fast, n)
    n += 1000


cines = {'Riocentro Ceibos': {'pelicula':{'nombre':'Coco','categoria':'infantil'},'reservas':25, 'asientos':30, 'tipo': '3D', 'horarios': ['10h30', '13h45', '15h30', '18h00']},
         'Riocentro Dorado': {'reservas':20, 'asientos':25, 'tipo': 'Digital', 'pelicula':{'nombre':'Jumanji', 'categoria':'aventura'}, 'horarios': ['12h30', '14h50', '18h30', '20h40']},
         'San Marino': {'reservas':24, 'asientos':32, 'tipo': '3D', 'pelicula':{'nombre':'JUMANJI', 'categoria':'aventura'}, 'horarios': ['11h30', '14h15', '19h30']},
         'Riocentro entre Ríos': {'reservas':32, 'asientos':32, 'tipo': 'Digital', 'pelicula':{'nombre':'Coco', 'categoria':'infantil'}, 'horarios': ['9h30', '12h15']},
         'Av. 6 diciembre': {'reservas':29, 'asientos':32, 'tipo': 'Digital', 'pelicula':{'nombre':'Starwars', 'categoria':'ficcion'}, 'horarios': ['17h30', '20h45', '23h30']},
         'Riocentro Norte': {'reservas':28, 'asientos':32, 'tipo': '3D', 'pelicula':{'nombre':'Implacable', 'categoria':'suspenso'}}}

def riocentro(cines):
    resultado = {}
    for k, v in cines.items():
        if "Riocentro" in k:
            ocupados = v.get("reservas")
            libres = v.get("asientos")-ocupados
            resultado.update({k.replace("Riocentro ",""):{"libres":libres, "ocupados":ocupados}})
    return resultado

print(riocentro(cines))

def peliculaporTipoExcepto(cines):
    peliculas = []
    for v in cines.values():
        categoria = v.get("pelicula").get("categoria")
        if(categoria!="infantil"):
            peliculas.append(v.get("pelicula").get("nombre").title())
    unicos = set(peliculas)
    return unicos

print(peliculaporTipoExcepto(cines))

def inasistencias10(cines):
    peliculas = []
    for v in cines.values():
        pelicula = v.get("pelicula").get("nombre").title()
        ocupados = v.get("reservas")
        total = v.get("asientos")
        libres = total-ocupados
        if(libres>total*0.1):
           peliculas.append(pelicula)
    return peliculas

print(inasistencias10(cines))


from django.shortcuts import render, redirect
from django.contrib.auth import logout
from django.views.generic import View
from .forms import ExtraDataForm
from django.core.mail import EmailMessage
class ExtraDataView(View):

	def get(self, request, *args, **kwargs):
		if request.user.status:
			return redirect('/')
		else:
			return render(request, 'users/extra_data.html')

	def post(self, request, *args, **kwargs):
		form = ExtraDataForm(request.POST)
		if form.is_valid():
			request.user.username = request.POST['username']
			request.user.email = request.POST['email']
			request.user.status = True
			request.user.save()
			send_email(request)
			return redirect('/')
		else:
			error_username = form['username'].errors.as_text()
			error_email = form['email'].errors.as_text()
			ctx = {'error_username':error_username, 'error_email':error_email}
			return render(request, 'users/extra_data.html', ctx)

def LogOut(request):
	logout(request)
	return redirect('/')

def send_email(request):

	msg = EmailMessage(subject = 'Bienvenida', from_email = 'Fabian Falon <fabian_falon@hotmail.com',
					   to = [request.user.email])

	msg.template_name = 'welcom'
	msg.template_content = {

	   'std_content00' : '
Hola %s Bienvenido a DevAsk
' % request.user
	}

	msg.send()


#!/usr/bin/env python
# coding=utf-8

import time
from threading import Thread

# 1.如果多个线程执行的都是同一个函数的话，各自之间不会有影响，各是各的

def test():
    print("------昨晚喝多了，下次少喝点------")
    time.sleep(1)

for i in range(5):
    t = Thread(target = test)
    t.start()

01-使用线程完成多任务.py


"""
    PREDICT COMPARE.py
        by Tim Müller

    This script tries to predict the optimal running time for a given
    application using different analytical models. For the input, it uses those
    as found in a benchmarking CSV, and shows us the difference between the
    predicted and actual runtimes.
"""

import argparse
import pandas as pd
import numpy as np
import os
import sys
from collections import defaultdict

DEFAULT_CSV = "benchmark_results.csv"
DEFAULT_MODEL_FOLDER = "analytical_models/"
DEFAULT_OUTPUT_FILE = "rankings.csv"

# Fill in the machine performance data please (as (peak_performance, peak_bandwidth, peak_performance_avx, peak_bandwidth_avx)) The bandwidth is over a single core.
# NOTE: hall GFLOP/s from https://www.intel.com/content/dam/support/us/en/documents/processors/APP-for-Intel-Core-Processors.pdf
# NOTE: all GB/s obtained with LIKWID (copy or copy-avx, 1GB load).
# NOTE: First two are non-simd, second two are yes-simd
# NOTE: GPU Stats from https://www.techpowerup.com/gpu-specs/geforce-rtx-2080.c3224 and https://www.techpowerup.com/gpu-specs/geforce-gtx-titan-x.c2632; beta_C_G is from https://www.gamersnexus.net/guides/2488-pci-e-3-x8-vs-x16-performance-impact-on-gpus
MACHINES = {
    "home_desktop": {
        1: (7.2, 14.93606, 28.8, 14.33228),
        2: (14.4, 15.86464, 57.6, 15.65240),
        4: (28.8, 16.50343, 115.2, 17.38215),
        8: (57.6, 16.94652, 230.4, 17.19413),
        16: (115.2, 17.09865, 460.8, 17.09826),
        32: (115.2, 17.01528, 460.8, 17.18359),
        "GPU": (10070.0, 448, 15.760)
    },
    "DAS5_1numa": {
        1: (2.4, 11.53074, 9.6, 11.42862),
        2: (4.8, 12.49312, 19.2, 12.27604),
        4: (9.6, 22.43777, 38.4, 20.62895),
        8: (19.2, 30.47322, 76.8, 27.52148),
        16: (38.4, 31.12496, 153.6, 28.11800),
        32: (76.8, 31.10337, 307.2, 28.10550)
    },
    "DAS5": {
        1: (2.4, 11.53074, 9.6, 11.42862),
        2: (4.8, 22.61996, 19.2, 21.99305),
        4: (9.6, 24.99948, 38.4, 24.37255),
        8: (19.2, 45.96919, 76.8, 41.58752),
        16: (38.4, 60.74498, 153.6, 55.02765),
        32: (76.8, 62.18000, 307.2, 56.00466),
        "GPU": (6691.0, 336.6, 15.760)
    },
    "macbook": {
        1: (5.8, 11.16392, 11.6, 11.26597),
        2: (11.6, 10.22710, 23.2, 10.38897),
        4: (23.2, 11.60071, 46.4, 11.72698),
        8: (23.2, 11.54814, 46.4, 11.68686),
        16: (23.2, 11.55648, 46.4, 11.67511),
        32: (23.2, 11.55118, 46.4, 11.57948)
    },
    "lisa": {
        1: (2.0, 13.21974, 16.0, 13.28313),
        2: (4.0, 25.49654, 32.0, 25.20818),
        4: (8.0, 47.30837, 64.0, 46.33363),
        8: (16.0, 47.05812, 128.0, 45.97598),
        16: (32.0, 60.49140, 256.0, 60.60499),
        32: (32.0, 65.63927, 256.0, 66.84047)
    }
}

def main(models_path, csv_files, output_path, filter_threads, show_all, show_none):
    print("\n### PREDICTION TOOL for ANALYTICAL MODELS ###\n")

    print(f"Tool configuration:")
    print(f" - Models folder       : {models_path}")
    if len(filter_threads) > 0: print(f" - Threads to display  : {', '.join(filter_threads)}")
    print(f" - Benchmark datafiles :")
    for csv in csv_files:
        print(f"    - '{csv}'")
    print("")

    print("Loading benchmark results... ", end="")
    sys.stdout.flush()
    data = pd.DataFrame()
    for csv in csv_files:
        csv_data = pd.read_csv(csv)
        data = pd.concat([data, csv_data], ignore_index=True)
    # Check if we got any
    if len(data) == 0:
        print(f"\n\nERROR: No benchmarks found in any of the given files.", file=sys.stderr)
        return -1
    print(f"Done (loaded {len(data)} benchmark(s))")

    print("Loading model implementations... ", end="")
    sys.stdout.flush()
    sys.path.append(models_path)
    models = {}
    for variation in data["variation"].unique():
        try:
            models[variation] = __import__(variation, fromlist=[variation])
        except ModuleNotFoundError:
            print(f"\n\nERROR: Could not load file '{variation}.py' from '{models_path}'.", file=sys.stderr)
            return -1
    print(f"Done (loaded {len(models)} models)")

    print("Reading benchmarks... ", end="")
    sys.stdout.flush()
    parameters = defaultdict(lambda: defaultdict(lambda: {"n": 0, "data": None}))
    for i, row in data.iterrows():
        # Get the parameter set as tuple
        hw_imp_pair = tuple(row.iloc[0:2]) + tuple([row.iloc[3]])
        # Stop if not filtered out
        if len(filter_threads) > 0 and str(hw_imp_pair[2]) not in filter_threads: continue
        parameter_set = tuple(row.iloc[4:10])
        if parameters[parameter_set][hw_imp_pair]["n"] == 0:
            parameters[parameter_set][hw_imp_pair]["data"] = np.array(row.iloc[10:])
        else:
            parameters[parameter_set][hw_imp_pair]["data"] += np.array(row.iloc[10:])
        parameters[parameter_set][hw_imp_pair]["n"] += 1
    # Once done, average all things
    parameters = {
        params: {
            hw_imp_pair: tuple(parameters[params][hw_imp_pair]["data"] / parameters[params][hw_imp_pair]["n"]) for hw_imp_pair in parameters[params]
        } for params in parameters
    }
    print("Done\n")

    if len(parameters) == 0:
        print("Nothing to do.\n")
        return 0

    print("Opening output file... ", end="")
    try:
        output = open(output_path, "w")
    except OSError as e:
        print(f"\nERROR: Could not open file \"{output_path}\": {e.strerror}\n")
        return -1
    print("Done")
    print("Writing CSV headers... ", end="")
    output.write("ranking_id,ranking_position,n_hidden_layers,nodes_per_layer,n_epochs,n_samples,sample_size,n_classes,predicted_machine,predicted_variation,predicted_n_threads,predicted_runtime,benchmarked_machine,benchmarked_variation,benchmarked_n_threads,benchmarked_runtime\n")
    print("Done\n")

    # Time to predict
    print("Predicting and comparing...")
    total = 0
    correct = 0
    hw_total = defaultdict(int)
    hw_incorrect = defaultdict(int)
    var_total = defaultdict(int)
    var_incorrect = defaultdict(int)
    thread_total = defaultdict(int)
    thread_incorrect = defaultdict(int)
    time_offset = defaultdict(lambda: defaultdict(list))
    for params in parameters:
        config_set = parameters[params]
        predicted_ranking = []
        benchmark_ranking = []
        for machine_id, variation, n_threads in config_set:
            # Predict the runtime for this ranking
            predicted_ranking.append([machine_id, variation, n_threads] + [runtime for runtime in models[variation].predict(params, MACHINES[machine_id], n_threads)])
            # Also add the runtimes for the benchmarking
            benchmark_ranking.append([machine_id, variation, n_threads] + [runtime for runtime in config_set[(machine_id, variation, n_threads)]])

        # Sort both rankings to actually rank them
        predicted_ranking = sorted(predicted_ranking, key=lambda elem: elem[3])
        benchmark_ranking = sorted(benchmark_ranking, key=lambda elem: elem[3])

        # Compare if the machines are the same
        matches = True
        for i in range(len(config_set)):
            machine_id = predicted_ranking[i][0]
            variation = predicted_ranking[i][1]
            n_threads = predicted_ranking[i][2]

            # Update the total
            hw_total[machine_id] += 1
            var_total[variation] += 1
            thread_total[n_threads] += 1

            # Compute and store the time offset
            offset = benchmark_ranking[i][3] / predicted_ranking[i][3]
            time_offset[(machine_id, n_threads)][variation].append(offset)

            if machine_id != benchmark_ranking[i][0] or variation != benchmark_ranking[i][1]:
                matches = False
                # Add an incorrect mark to both the hardware and the incorrect
                hw_incorrect[machine_id] += 1
                var_incorrect[variation] += 1
                thread_incorrect[n_threads] += 1

            # Print the ranking
            output.write(f"{total},{i},{','.join([str(p) for p in params])},{machine_id},{variation},{n_threads},{predicted_ranking[i][3]},{benchmark_ranking[i][0]},{benchmark_ranking[i][1]},{benchmark_ranking[i][2]},{benchmark_ranking[i][3]}\n")

        # If they aren't, let the user know
        if not show_none and (show_all or not matches):
            # Print that they differ
            line_size = 60
            print(f"\nResults for parameters {params} (id {total}) differ:")
            print("  Predicted:" + " " * (line_size - 6) + "Benchmarked:")
            for i in range(len(config_set)):
                n_threads = predicted_ranking[i][2]
                text = f"   {i + 1}) {predicted_ranking[i][0].upper()} ({n_threads} thread{'' if n_threads == 1 else 's'}) with {predicted_ranking[i][1].upper()} ({predicted_ranking[i][3]:.4f}s)"
                text += " " * (line_size - len(text)) + " VS "
                n_threads = benchmark_ranking[i][2]
                text += f"   {i + 1}) {benchmark_ranking[i][0].upper()} ({n_threads} thread{'' if n_threads == 1 else 's'}) with {benchmark_ranking[i][1].upper()} ({benchmark_ranking[i][3]:.4f}s)"
                print(text)
            print("")

        # Update the accuracy score
        total += 1
        correct += 1 if matches else 0

    # Convert time_offset in a normal dictionary
    time_offset = {m: {v: time_offset[m][v].copy() for v in time_offset[m]} for m in time_offset}

    print("\n***STATS***\n")
    print("Ranking accuracy:")
    print(f" - Overall accuracy: {(correct / total * 100):.2f}%")
    print(" - Accuracy per machine:")
    for m in hw_total:
        print(f"    - {m} : {(hw_total[m] - hw_incorrect[m]) / hw_total[m] * 100:.2f}%")
    print(" - Accuracy per variation:")
    for v in var_total:
        print(f"    - {v} : {(var_total[v] - var_incorrect[v]) / var_total[v] * 100:.2f}%")
    print(" - Accuracy per amount of threads:")
    for t in thread_total:
        print(f"    - {t} : {(thread_total[t] - thread_incorrect[t]) / thread_total[t] * 100:.2f}%")

    print("\nRatio of prediction time to benchmark time per machine (number of threads), per variation:")
    for m, t in time_offset:
        print(f" - {m} ({t} threads):")
        for v in time_offset[(m, t)]:
            offsets = time_offset[(m, t)][v]
            print(f"    - {v}:")

            # Compute the mean first
            mean = sum(offsets) / len(offsets)
            print(f"       - mean     = {mean:.2f}x")

            # Then, the variance
            var = sum([(mean - offset) ** 2 for offset in offsets]) / len(offsets)
            print(f"       - variance = {var:.2f}")

    print("\nDone.\n")

    output.close()
    return 0

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument("-o", "--output", default=DEFAULT_OUTPUT_FILE, required=False, help=f"Path the output file to which the rankings will be written. (Default: {DEFAULT_OUTPUT_FILE})")
    parser.add_argument("-c", "--csvs", nargs='+', required=True, help=f"Files to load the benchmarks from. Can be more than one, which will be pasted internally after one another.")
    parser.add_argument("-M", "--models", default=DEFAULT_MODEL_FOLDER, required=False, help=f"Path to the folder containing the model implementations. (Default: {DEFAULT_MODEL_FOLDER})")
    parser.add_argument("-a", "--all", action="store_true", help="If given, displays all rankings instead of just the incorrect ones.")
    parser.add_argument("-s", "--silent", action="store_true", help="If given, displays no rankings at all (overrides --all).")
    parser.add_argument("-t", "--threads", default=[], nargs='+', help="The numbers of threads to keep. All other n_threads will be filtered out. If omitted, displays all threads")

    args = parser.parse_args()

    # Check the validity of the CSV files
    csvs = []
    while len(args.csvs) > 0:
        csv = args.csvs[0]
        args.csvs = args.csvs[1:]

        if not os.path.exists(csv):
            print(f"ERROR: File '{csv}' does not exist.", file=sys.stderr)
            exit(-1)
        if os.path.isdir(csv):
            # Append all this folder's children to the list
            csvs += [os.path.join(csv, f) for f in os.listdir(csv)]
            continue
        elif not os.path.isfile(csv):
            print(f"ERROR: File '{csv}' is not a file.", file=sys.stderr)
            exit(-1)

        # Append the file
        csvs.append(csv)

    # Check if the models folder exists
    if not os.path.exists(args.models):
        print(f"ERROR: Folder '{args.models}' does not exist.", file=sys.stderr)
        exit(-1)
    if not os.path.isdir(args.models):
        print(f"ERROR: Folder '{args.models}' is not a folder.", file=sys.stderr)
        exit(-1)

    # So far so good, let's run main
    exit(main(args.models, csvs, args.output, args.threads, args.all, args.silent))


import yfinance as yf
from datetime import datetime

def comp_det():
    my_file = open("competitors.txt", "r")
    content = my_file.read()
    content_list = content.split("\n")
    my_file.close()
    return content_list
def convert_tocsv(price_history,i,company_name):
    path_out='/home/aakash/data_pipeline/'
    now = datetime.now()
    dt_string = now.strftime("%d-%m-%Y %H:%M:%S")
    price_history.to_csv(path_out+company_name+'_'+'Nasdaq'+'_'+i+'_'+dt_string+'.csv')

l=comp_det()
for i in l:
    l=yf.Ticker(i)
    company_name = l.info['longName']
    price_history = yf.Ticker(i).history(period='5d')# we can scale this paramter as per the requirement
    convert_tocsv(price_history,i,company_name)


# -*- coding: utf-8 -*-

"""
@author: tjm
@software: PyCharm
@file: detect.py
@time: 2021/9/16 9:48
"""

import os
import sys
import skimage.io

from app import app
from app.mrcnn.config import Config
from datetime import datetime
from pathlib import Path
import cv2

# Root directory of the project
ROOT_DIR = os.getcwd()

# Import Mask RCNN
sys.path.append(ROOT_DIR)  # To find local version of the library
# from mrcnn import utils
from app.mrcnn import model as modellib
from app.mrcnn import visualize

# Import COCO config
# sys.path.append(os.path.join(ROOT_DIR, "samples/coco/"))  # To find local version
# from samples.coco import coco

# Directory to save h5 and trained model

MODEL_DIR = os.path.join(ROOT_DIR, "app\mrcnn\h5")

# Local path to trained weights file
COCO_MODEL_PATH = os.path.join(MODEL_DIR, "shapes20210726T1823/mask_rcnn_shapes_0014.h5")
# Download COCO trained weights from Releases if needed
if not os.path.exists(COCO_MODEL_PATH):
    # utils.download_trained_weights(COCO_MODEL_PATH)
    print("junlintianxia************ h5 model not exist ***********")

# Directory of images to run detection on
IMAGE_DIR = os.path.join(ROOT_DIR, "images")

class ShapesConfig(Config):
    """Configuration for training on the toy shapes dataset.
    Derives from the base Config class and overrides values specific
    to the toy shapes dataset.
    """
    # Give the configuration a recognizable name
    NAME = "shapes"

    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each
    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).
    GPU_COUNT = 1
    IMAGES_PER_GPU = 1

    # Number of classes (including background)
    NUM_CLASSES = 1 + 12  # background + 12 shapes

    # Use small images for faster training. Set the limits of the small side
    # the large side, and that determines the image shape.
    IMAGE_MIN_DIM = 80
    IMAGE_MAX_DIM = 512

    # Use smaller anchors because our image and objects are small
    # RPN_ANCHOR_SCALES = (8 * 6, 16 * 6, 32 * 6, 64 * 6, 128 * 6)  # anchor side in pixels
    RPN_ANCHOR_SCALES = (8 * 2, 16 * 2, 32 * 2, 64 * 2, 128 * 2)

    # Reduce training ROIs per image because the images are small and have
    # few objects. Aim to allow ROI sampling to pick 33% positive ROIs.
    TRAIN_ROIS_PER_IMAGE = 10

    # Use a small epoch since the data is simple
    STEPS_PER_EPOCH = 10

    # use small validation steps since the epoch is small
    VALIDATION_STEPS = 5

# import train_tongue
# class InferenceConfig(coco.CocoConfig):
class InferenceConfig(ShapesConfig):
    # Set batch size to 1 since we'll be running inference on
    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
    GPU_COUNT = 1
    IMAGES_PER_GPU = 1

# 检测图片
class detect_image():

    def __init__(self, imgPath=None):
        self.imgPath = imgPath
        self.config = InferenceConfig()
        # Create model object in inference mode.
        self.model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=self.config)

        # Load weights trained on MS-COCO
        self.model.load_weights(COCO_MODEL_PATH, by_name=True)

        # COCO Class names
        # Index of the class in the list is its ID. For example, to get ID of
        # the teddy bear class, use: class_names.index('teddy bear')
        self.class_names = ['BG', "AJ", "BX", "CJ", "CK", "CR", "FZ", "JG", "PL", "QF", "TJ", "ZC", "ZW"]
        # Load a random image from the images folder
        # file_names = next(os.walk(IMAGE_DIR))[2]
        # image = skimage.io.imread("./images/CJ6798.jpg")
        self.image = skimage.io.imread(imgPath)

    def call(self):
        a = datetime.now()
        # Run detection
        results = self.model.detect([self.image], verbose=1)
        b = datetime.now()
        # Visualize results
        print("time: ", (b - a).seconds)
        r = results[0]

        print("=======", r)

        visualize.display_instances(self.image, r['rois'], r['masks'], r['class_ids'],
                                    self.class_names, r['scores'])

# 检测视频
class detect_video():

    def __init__(self, videoPath=None):
        self.videoPath = videoPath
        self.video_capture = cv2.VideoCapture(self.videoPath)

        self.config = InferenceConfig()
        # Create model object in inference mode.
        self.model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=self.config)

        # Load weights trained on MS-COCO
        self.model.load_weights(COCO_MODEL_PATH, by_name=True)

        # COCO Class names
        # Index of the class in the list is its ID. For example, to get ID of
        # the teddy bear class, use: class_names.index('teddy bear')
        self.class_names = ['BG', "AJ", "BX", "CJ", "CK", "CR", "FZ", "JG", "PL", "QF", "TJ", "ZC", "ZW"]

    def do_detect(self):
        a = datetime.now()
        # Run detection
        # TODO 动态创建保存路径，保存成功后，更新视频数据库缺陷文件夹路径（需修改数据库表，增加是否检测，检测结果路径两个字段）
        # TODO 动态创建路径已经实现，现在需要实现的是修改数据库，增加是否检测字段与检测结果路径字段
        # 获取文件名，当做文件保存路径名
        video_name = os.path.splitext(os.path.basename(self.videoPath))[0]
        # 拼成文件要保存的路径，文件直接保存在项目根目录results文件夹中，如果要保存在模型result文件夹中，需要在 visualize.display_instances中上进行设置
        folder_path = os.path.join('results', str(video_name))
        # save_path = os.path.join(ROOT_DIR, folder_path)
        # 先判断是否已经存在文件夹，若不存在，则新建新的文件夹
        # 若文件夹已经存在，则删除文件夹内的所有数据（用户点击重新检测的情况）
        if not Path(folder_path).exists():
            os.makedirs(folder_path)
            print('文件存储路径', folder_path)
        else:
            files = os.listdir(folder_path)
            for file in files:
                c_path = os.path.join(folder_path, file)
                os.remove(c_path)
        while True:
            ret, frame = self.video_capture.read()
            if ret:
                # TODO 常识转换灰度图，对比性能
                results = self.model.detect([frame], verbose=1)
                b = datetime.now()
                # Visualize results
                print("time: ", (b - a).seconds)
                r = results[0]

                print("=======", r)
                cur_class_id = len(r['class_ids'])
                if cur_class_id:
                    # TODO 方法visualize.display_instances需增加一个参数：缺陷结果保存路径
                    # 新增一个参数，file_path， 参数为想要保存的缺陷结果路径。该路径在上面设置。
                    visualize.display_instances(frame, r['rois'], r['masks'], r['class_ids'],
                                                self.class_names, folder_path, r['scores'])
            else:
                break
        return folder_path

if __name__ == "__main__":
    # det = detect_image("./images/CJ6798.jpg")
    # det.call()

    root_path = app.config['UP_DIR']  # 文件上传保存路径
    det = detect_video(
        r"D:\python-workspace\FlaskVideo-master\app\static\video\21-09-13\2202109131608391c1babe4b2204fd0a9fca5e0c1db731d.mp4")
    det.do_detect()


import unittest
from models.rectangle import Rectangle

class TestRectangle(unittest.TestCase):
    def test_attributes(self):
        r1 = Rectangle(10, 15, 8, 9, 1)
        self.assertEqual(r1.width, 10)
        self.assertEqual(r1.height, 15)
        self.assertEqual(r1.x, 8)
        self.assertEqual(r1.y, 9)
        self.assertEqual(r1.id, 1)

    def test_area(self):
        r2 = Rectangle(3, 4)
        self.assertEqual(r2.area(), 12)

    def test_update(self):
        r1 = Rectangle(10, 15, 8, 9, 1)
        r1.update(width=12)
        self.assertEqual(r1.width, 12)

    def test_to_dictionary(self):
        r1 = Rectangle(10, 15, 8, 9, 1)
        d1 = r1.to_dictionary()
        self.assertEqual(
            d1, {'id': 1, 'width': 10, 'height': 15, 'x': 8, 'y': 9})

    def test_validation_errors(self):
        with self.assertRaises(TypeError):
            Rectangle("10", 10)
        with self.assertRaises(ValueError):
            Rectangle(-10, 10)
        with self.assertRaises(TypeError):
            Rectangle(10, "10")
        with self.assertRaises(ValueError):
            Rectangle(10, -10)
        with self.assertRaises(TypeError):
            Rectangle(10, 10, "10")
        with self.assertRaises(ValueError):
            Rectangle(10, 10, -10)
        with self.assertRaises(TypeError):
            Rectangle(10, 10, 10, "10")
        with self.assertRaises(ValueError):
            Rectangle(10, 10, 10, -10)

    def test_str_representation(self):
        """Test the string representation of the Rectangle"""
        r1 = Rectangle(10, 15, 8, 9, 1)
        self.assertEqual(str(r1), "[Rectangle] (1) 8/9 - 10/15")

if __name__ == "__main__":
    unittest.main()


# 소인수 분해
import random

# 소수
def prime(n):
    isPrime = True
    for i in range(2, n):
        if n % i == 0:
            isPrime = False

    return isPrime

def Factorization(n):
    div = 2
    val = n
    primes = []

    while val >= div:
        if val % div == 0:
            val = val / div
            primes.append(div)
        else:
            div += 1

    print(f'{n} 소인수분해 결과 : {primes}')

    prime = 0
    for i in primes:
        if i != prime:
            print(f'{i}의 지수 : {primes.count(i)}')
            prime = i

n = random.randint(100, 1000)
Factorization(n)


#!/usr/bin/env python3
from sys import stdin, exit
from copy import *
from heapq import *
from collections import *
from itertools import *
from math import *
from z3 import *

data = open("input.txt").read().strip().split('\n')
#data = stdin.read().split('\n')[:-1]

def run(inp):
    var = {'x': IntVal(0), 'y': IntVal(0), 'z': IntVal(0), 'w': IntVal(0)}
    #var = {'x': 0, 'y': 0, 'z': 0, 'w': 0}
    cur = 0
    for line in data:
        p = line.split(' ')
        if p[0] == 'inp':
            var[p[1]] = inp[cur]
            #var[p[1]] = int(inp[cur])
            cur += 1
        else:
            if p[2] in var:
                op2 = var[p[2]]
            else:
                op2 = int(p[2])
            if p[0] == 'add':
                var[p[1]] += op2
            elif p[0] == 'mul':
                var[p[1]] *= op2
            elif p[0] == 'div':
                var[p[1]] = simplify(If(var[p[1]] < 0, -((-var[p[1]]) / op2), var[p[1]] / op2))
                #if var[p[1]] < 0:
                #    var[p[1]] = -((-var[p[1]]) // op2)
                #else:
                #    var[p[1]] = var[p[1]] // op2
            elif p[0] == 'mod':
                var[p[1]] %= op2
            elif p[0] == 'eql':
                var[p[1]] = If(var[p[1]] == op2, 1, 0)
                #var[p[1]] = 1 if var[p[1]] == op2 else 0
    return var['z']

magicIn = [Int(f'in{i}') for i in range(14)]
s = Solver()
for i in magicIn:
    s.add(i >= 1)
    s.add(i <= 9)
x = run(magicIn)
x = simplify(x)
print(simplify(x == 0))
s.add(x == 0)

print(s)
print(s.check())

while s.check() == sat:
    m = s.model()
    print(m)


def FactDiff(value):
    fact=1
    sum=0
    sum1=0
    sum2=0
    for i in range(1,value,1):
        if((value%i)==0):
            sum=sum+i
            sum1=sum1+i
        else:
            return sum1-sum

def main():

    ret=0

    print("Enter the number")
    no=int(input())

    ret=FactDiff(no)
    print(ret)

if __name__=="__main__":
    main()


from django.http import HttpResponse
from django.shortcuts import render
# def index(request):
#     return HttpResponse("
Hello World
")
#
# def about(request):
#     return HttpResponse("
This is the about page
")

def index(request):
    # params = {'name':'Harry', 'place': 'Earth'}
    return render(request,"index.html")
    #return HttpResponse("Home")

def analyze(request):
    djtext = request.POST.get('text', 'default')
    removepunc = request.POST.get('removepunc', 'off')
    fullcaps = request.POST.get('fullcaps', 'off')
    newlineremover = request.POST.get('newlineremover', 'off')
    extraspaceremover = request.POST.get('extraspaceremover', 'off')
    charcount = request.POST.get('Charcount', 'off')
    # print(removepunc)
    # print(djtext)
    # analyzetext = djtext
    punctuations = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
    analyzed = ""
    if removepunc == 'on':
        for char in djtext:
            if char not in punctuations:
                analyzed = analyzed + char
        djtext = analyzed
    if fullcaps == 'on':
        analyzed = ""
        for char in djtext:
            analyzed = analyzed+char.upper()
        djtext=analyzed
    if newlineremover == 'on':
        analyzed = ""
        for char in djtext:
            if char != '\n' and char != '\r':
                analyzed = analyzed + char
        djtext=analyzed
    if extraspaceremover == 'on':
        analyzed = ""
        for index,char in enumerate(djtext):
            if djtext[index] == " " and djtext[index+1] == " ":
                pass
            else:
                analyzed = analyzed + char
        djtext=analyzed
    if charcount == 'on':
        count = 0
        analyzed = ""
        for char in djtext:
                count= count+1
        return HttpResponse("Character count = "+str(count))
    if removepunc=='off' and fullcaps=='off' and newlineremover == 'off' and extraspaceremover=='off' and charcount=='off':
        return HttpResponse("Error!")

    params = {'purpose':'Remove punctuation', 'analyzed_text': analyzed}
    return render(request, "analyze.html", params)

# def capfirst(request):
#     return HttpResponse("Capitalize first")
#
# def newlineremove(request):
#     return HttpResponse("New Line remover")
#
# def spaceremove(request):
#     return HttpResponse("Space remover")

# def charcount(request):
#     return HttpResponse("Character count")


import os
import sys
import pickle
import random
import datetime
import argparse
import numpy as np
import matplotlib.pyplot as plt
from scipy import misc

from PIL import Image

def predict(image, SRCNN, epoch, mode='compare'):
    scale = 3.0
    input_size = 33
    label_size = 21
    pad = int((input_size - label_size) / 2)

    w, h, c = image.shape
    w -= int(w % scale)
    h -= int(h % scale)
    image = image[0:w, 0:h, :]

    scaled = misc.imresize(image, 1.0/scale, 'bicubic')
    scaled = misc.imresize(scaled, scale/1.0, 'bicubic')/255.0
    newimg = np.zeros(scaled.shape)

    for i in range(0, h - input_size + 1, label_size):
        for j in range(0, w - input_size + 1, label_size):
            sub_img_y = np.expand_dims(scaled[j : j + input_size, i : i + input_size, 0],axis=-1)
            prediction = np.squeeze(SRCNN.predict(np.expand_dims(sub_img_y, axis=0)))
            prediction[prediction > 1.0] = 1.0
            newimg[j + pad : j + pad + label_size, i + pad : i + pad + label_size, 0] = prediction
    newimg[...,1:] = scaled[...,1:]
    newimg = newimg[pad : w - w % input_size, pad : h - h % input_size,:]

    if(mode == 'compare'):
        height = int(1.2*h)
        width = int(1.2*w)
        image = pad_img(ycbcr2rgb(image*255.0), [width,height])
        scaled = pad_img(ycbcr2rgb(scaled*255.0), [width,height])
        newimg = pad_img(ycbcr2rgb(newimg*255.0), [width,height])
        saveimg = np.hstack((image,scaled,newimg))
        # plt.subplot(131)
        # plt.imshow(image/255.0)
        # plt.subplot(132)
        # plt.imshow(scaled/255.0)
        # plt.subplot(133)
        # plt.imshow(newimg/255.0)
        # plt.show()

        array2image(saveimg).save('../TrainResult/Round{}.png'.format(epoch+1))
        return None
    else:
        return newimg

def pad_img(img, img_size):
    img_h,img_w=img.shape[0], img.shape[1]
    target_h,target_w=img_size[0],img_size[1]

    if len(img.shape)==3:
        d=img.shape[2]
        padded=np.zeros((target_h, target_w,d))
    elif len(img.shape)==2:
        padded=np.zeros((target_h, target_w))

    padded[(target_h-img_h)//2:(target_h-img_h)//2+img_h,(target_w-img_w)//2:(target_w-img_w)//2+img_w,...]=img

    return padded

def array2image(arr):
    if(np.max(arr)<=1):
        image = Image.fromarray((arr*255).astype(np.uint8))
    else:
        image = Image.fromarray((arr).astype(np.uint8))

    return image

def showFLAGS(FLAGS):
    for i in vars(FLAGS):
        if len(i) < 8:
            print(i + '\t\t------------  ' + str(vars(FLAGS)[i]))
        else:
            print(i + '\t------------  ' + str(vars(FLAGS)[i]))
    print()

def progressbar(progress, total, length=40):
    total = total - 1
    num = int(progress/total*length)
    sys.stdout.write('#' * num + '_' * (length - num) )
    sys.stdout.write(':{:.2f}%'.format(progress/total*100)+ '\r')
    if progress == total:
        sys.stdout.write('\n\n')
    sys.stdout.flush()

def checkpath(path):
    try:
        os.makedirs(path)
        # print('creat ' + path)
    except OSError:
        pass

def log(text):
    """
    log status with time label
    """
    print()
    nowTime=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    line1 = '='*10 + '  ' + nowTime + '  '+ '='*10
    length = len(line1)
    leftnum = int((length - 4 - len(text))/2)
    rightnum = length - 4 - len(text) - leftnum
    line2 = '*'*leftnum + ' '*2 + text  + ' '*2 + '*'*rightnum
    print(line1)
    print(line2)
    print('='*len(line1))

def all_files_under(path, extension=None, append_path=True, sort=True):
    if append_path:
        if extension is None:
            filenames = [os.path.join(path, fname) for fname in os.listdir(path)]
        else:
            filenames = [os.path.join(path, fname) for fname in os.listdir(path) if fname.endswith(extension)]
    else:
        if extension is None:
            filenames = [os.path.basename(fname) for fname in os.listdir(path)]
        else:
            filenames = [os.path.basename(fname) for fname in os.listdir(path) if fname.endswith(extension)]

    if sort:
        filenames = sorted(filenames)

    return filenames

mat = np.array(
    [[ 65.481, 128.553, 24.966 ],
     [-37.797, -74.203, 112.0  ],
     [  112.0, -93.786, -18.214]])
mat_inv = np.linalg.inv(mat)
offset = np.array([16, 128, 128])

def rgb2ycbcr(rgb_img):
    ycbcr_img = np.zeros(rgb_img.shape)
    for x in range(rgb_img.shape[0]):
        for y in range(rgb_img.shape[1]):
            ycbcr_img[x, y, :] = np.round(np.dot(mat, rgb_img[x, y, :] * 1.0 / 255) + offset)
    return ycbcr_img
def ycbcr2rgb(ycbcr_img):
    rgb_img = np.zeros(ycbcr_img.shape, dtype=np.uint8)
    for x in range(ycbcr_img.shape[0]):
        for y in range(ycbcr_img.shape[1]):
            [r, g, b] = ycbcr_img[x,y,:]
            rgb_img[x, y, :] = np.maximum(0, np.minimum(255, np.round(np.dot(mat_inv, ycbcr_img[x, y, :] - offset) * 255.0)))
    return rgb_img


import os
from pathlib import Path

def delete_dir_tree(directory: Path) -> None:
    """
    Recursively delete a whole directory and its content.
    Since there is no built-in function for this in the pathlib API and we want to keep it consistent, we have to use a self-written function.

    :param directory: The directory that should be removed
    """

    dir = directory

    for file in dir.iterdir():
        if file.is_dir():
            delete_dir_tree(file)
        else:
            file.unlink()
    dir.rmdir()

def pf(calling_class, file_path: str) -> str:
    """

    :param calling_class: the class of which this method is called
    :param file_path: path to file
    :return: joined path
    """
    return os.path.join(calling_class.path, file_path)