#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
LOAD QUESTIONIRES

Created on Fri Aug 29 13:20:15 2022

@author: Sophia
"""
import pandas as pd
import numpy as np
import load_utils as lu

import seaborn as sns
import matplotlib.pyplot as plt

# Import data sheets to python with pandas

#%%%
""" LOADING questionaires"""
'''The column "pizza" is the nr. meesurements mox used in the office, and is
    bellow manually connected with office ID.'''

# Load in building  checklists from excel to pandas
q_101 = pd.read_excel('../DATA/01_Denmark/BuildingID_101/03_Questionnaire/Copy_101_Winter2020_Questionnaire.xlsx')
q_102 = pd.read_excel('../DATA/01_Denmark/BuildingID_102/03_Questionnaire/Copy_102_Winter2020_Questionnaire.xlsx')
q_103 = pd.read_excel('../DATA/01_Denmark/BuildingID_103/03_Questionnaire/Copy103_Spring2021_Questionnaire.xlsx')

q_103['officeID'] = q_103['pizza'].map({1:'103a', 3:'103a',  8:'103a', 2:'103b', 25:'103b',
                                        7:'103c', 22:'103c', 12:'103d' })

q_103E = pd.read_excel('../DATA/01_Denmark/BuildingID_103F/03_Questionnaire/103_F2021_Questionnaire.xlsx')
q_103E['officeID'] = q_103E['pizza'].map({10:'103_E2021a', 11:'103_E2021a', 13:'103_E2021b',12:'103_E2021c'})

q_104 = pd.read_excel('../DATA/01_Denmark/BuildingID_104/03_Questionnaire/104_Q.xlsx')
q_104['officeID'] = q_104['pizza'].map({22:'104a', 24:'104b'})

q_105 = pd.read_excel('../DATA/01_Denmark/BuildingID_105/03_Questionnaire/105_Q.xlsx')
q_105['officeID'] = q_105['pizza'].map({22:'105a', 24:'105a',10:'105a'})

q_301 = pd.read_excel('../DATA/01_Denmark/BuildingID_301/03_Questionnaire/Copy301_Spring2021_Questionnaire.xlsx')
q_301['officeID'] = q_301['pizza'].map({16:'301a', 3:'301b', 14:'301c', 13:'301d',
                                        15:'301e', 12:'301f', 17:'301g' })

q_302 = pd.read_excel('../DATA/01_Denmark/BuildingID_302/03_Questionnaire/Copy302_Spring2021_Questionnaire.xlsx')
q_302['officeID'] = q_302['pizza'].map({5:'302a', 2:'302b', 21:'302c', 8:'302d', 11:'302e'})

q_303 = pd.read_excel('../DATA/01_Denmark/BuildingID_303/03_Questionnaire/Copy303_Spring2021_Questionnaire.xlsx')
q_303['officeID'] = q_303['pizza'].map({15:'303a', 12:'303b', 17:'303c', 13:'303d', 14:'303d'})

q_304 = pd.read_excel('../DATA/01_Denmark/BuildingID_304/03_Questionnaire/Copy304_Spring2021_Questionnaire.xlsx')
q_304['officeID'] = q_304['pizza'].map({16:'304a', 3:'304b', 14:'304c', 13:'304d',
                                        15:'304e', 12:'304f', 17:'304g' })

q_301E = pd.read_excel('../DATA/01_Denmark/BuildingID_301_E20/03_Questionnaire/301_E2021_Questionnaire.xlsx')
q_301E['officeID'] = q_301E['pizza'].map({8:'301_E2021a', 22:'301_E2021b', 23:'301_E2021c', 17:'301_E2021d'})

q_305A = pd.read_excel('../DATA/01_Denmark/BuildingID_305A/03_Questionnaire/305A_Q.xlsx')
q_305A['officeID'] = q_305A['pizza'].map({1:'305b', 2:'305c',3:'305d',4:'305e',5:'305f',
                                          6:'305g', 7:'305h', 9:'305i'})

q_306 = pd.read_excel('../DATA/01_Denmark/BuildingID_306/03_Questionnaire/306_F2021_Questionnaire.xlsx')
q_306['officeID'] = q_306['pizza'].map({13:'3060A', 11:'3060B',12:'3061A',10:'3061A'})

q_307 = pd.read_excel('../DATA/01_Denmark/BuildingID_307/03_Questionnaire/307_Fall2021_Questionnaire.xlsx')
q_307['officeID'] = q_307['pizza'].map({8:'307a', 17:'307a',22:'307a',23:'307a'})

q_308 = pd.read_excel('../DATA/01_Denmark/BuildingID_307/03_Questionnaire/307_Fall2021_Questionnaire.xlsx')
q_308['officeID'] = q_308['pizza'].map({10:'308Ab', 22:'308Ab',
                                        11:'308Ba',12:'308Ba',13:'308Bb'})

q_309 = pd.read_excel('../DATA/01_Denmark/BuildingID_309/03_Questionnaire/309_Q.xlsx')
q_309['officeID'] = q_309['pizza'].map({4:'309a', 7:'309a',2:'309b',15:'309b',17:'309c'})

q_310 = pd.read_excel('../DATA/01_Denmark/BuildingID_310/03_Questionnaire/310_Q.xlsx')
q_310['officeID'] = q_310['pizza'].map({18:'310a', 19:'310a',16:'310b'})

q_311 = pd.read_excel('../DATA/01_Denmark/BuildingID_311/03_Questionnaire/311_Q.xlsx')
q_311['officeID'] = q_311['pizza'].map({11:'311a', 12:'311a',8:'311b'})

q_312 = pd.read_excel('../DATA/01_Denmark/BuildingID_312/03_Questionnaire/312_Q.xlsx')
q_312['officeID'] = q_312['pizza'].map({20:'312a', 21:'312a',23:'312b',25:'312c'})

q_313 = pd.read_excel('../DATA/01_Denmark/BuildingID_313/03_Questionnaire/313_Q.xlsx')
q_313['officeID'] = q_313['pizza'].map({11:'313a', 12:'313a',13:'313a',
                                        14:'313b',16:'313b',
                                        18:'313c',19:'313c',
                                        8:'313d', 20:'313e',
                                        23:'313f',25:'313f'})

q_401 = pd.read_excel('../DATA/02_Greenland/Building_ID_401/03_Questionnaire/Copy_401_Spring2021_Questionnaire.xlsx')

q_402 = pd.read_excel('../DATA/02_Greenland/Building_ID_402/03_Questionnaire/402_Spring2021_Questionnaire.xlsx')
q_402['officeID'] = q_402['pizza'].map({8:'402b', 11:'402e', 4:'402d', 9:'402a', 10:'402c' })

q_403 = pd.read_excel('../DATA/02_Greenland/Building_ID_403/03_Questionnaire/403_Spring2021_Questionnaire.xlsx')
q_403['officeID'] = q_403['pizza'].map({13:'403a', 12:'403b', 14:'403c', 2:'403d' })

q_404 = pd.read_excel('../DATA/02_Greenland/Building_ID_404/03_Questionnaire/404_Spring2021_Questionnaire.xlsx')
q_404['officeID'] = q_404['pizza'].map({20:'404a', 19:'404b'})

q_405 = pd.read_excel('../DATA/02_Greenland/Building_ID_405/03_Questionnaire/405_Spring2021_Questionnaire.xlsx')
q_405['officeID'] = q_405['pizza'].map({17:'405a', 21:'405b',18:'405c',1:'405d'})

q_407 = pd.read_excel('../DATA/02_Greenland/Building_ID_407/03_Questionnaire/407_Spring2021_Questionnaire.xlsx')
q_407['officeID'] = q_407['pizza'].map({18:'407a', 19:'407b',17:'407c'})

q_409 = pd.read_excel('../DATA/02_Greenland/Building_ID_409/03_Questionnaire/409_Spring2021_Questionnaire.xlsx')
q_409['officeID'] = q_409['pizza'].map({9:'409a', 11:'409b',12:'409c'})

q_410 = pd.read_excel('../DATA/02_Greenland/Building_ID_410/03_Questionnaire/410_Spring2021_Questionnaire.xlsx')
q_410['officeID'] = q_410['pizza'].map({3:'410a', 2:'410b',4:'410c'})

q_411 = pd.read_excel('../DATA/02_Greenland/Building_ID_411/03_Questionnaire/411_Spring2021_QuestionnaireKopi.xlsx')
q_411['officeID'] = q_411['pizza'].map({6:'411a', 5:'411b',8:'411c'})

q_412 = pd.read_excel('../DATA/02_Greenland/Building_ID_412/03_Questionnaire/412_Spring2021_QuestionnaireKopi.xlsx')
q_412['officeID'] = q_412['pizza'].map({20:'412a', 23:'412b',21:'412c'})

q_413 = pd.read_excel('../DATA/02_Greenland/Building_ID_413/03_Questionnaire/413_Spring2021_Questionnaire.xlsx')
q_413['officeID'] = q_413['pizza'].map({29:'413a', 33:'413b',35:'413c'})

q_414 = pd.read_excel('../DATA/02_Greenland/Building_ID_414/03_Questionnaire/414_Spring2021_Questionnaire.xlsx')
q_414['officeID'] = q_414['pizza'].map({31:'414a', 32:'414b',24:'414c'})

q_415 = pd.read_excel('../DATA/02_Greenland/Building_ID_415/03_Questionnaire/415_Spring2021_Questionnaire.xlsx')
q_415['officeID'] = q_415['pizza'].map({30:'415a', 25:'415b',28:'415c'})

q_416 = pd.read_excel('../DATA/02_Greenland/Building_ID_416/03_Questionnaire/416_Spring2021_Questionnaire.xlsx')
q_416['officeID'] = q_416['pizza'].map({34:'416a',26:'416b',27:'416c'})

q_417 = pd.read_excel('../DATA/02_Greenland/Building_ID_417/03_Questionnaire/417_Spring2021_Questionnaire.xlsx')
q_417['officeID'] = q_417['pizza'].map({7:'417a',10:'417b',13:'417c'})

q_418 = pd.read_excel('../DATA/02_Greenland/Building_ID_418/03_Questionnaire/418_Spring2021_Questionnaire.xlsx')
q_418['officeID'] = q_418['pizza'].map({7:'418a',10:'418b',13:'418c'})

q_419 = pd.read_excel('../DATA/02_Greenland/Building_ID_419/03_Questionnaire/419_Spring2021_Questionnaire.xlsx')
q_419['officeID'] = q_419['pizza'].map({9:'419a',8:'419b',3:'419c'})

q_420 = pd.read_excel('../DATA/02_Greenland/Building_ID_420/03_Questionnaire/420_Spring2021_Questionnaire.xlsx')
q_420['officeID'] = q_420['pizza'].map({2:'420a',5:'420b',4:'420c'})

q_421 = pd.read_excel('../DATA/02_Greenland/Building_ID_421/03_Questionnaire/Copy_421_Spring2021_Questionnaire.xlsx')
q_421['officeID'] = q_421['pizza'].map({17:'421a',18:'421b',19:'421c'})

q_422 = pd.read_excel('../DATA/02_Greenland/Building_ID_422/03_Questionnaire/Copy_422_Spring2021_Questionnaire.xlsx')
q_422['officeID'] = q_422['pizza'].map({22:'422a',1:'422b',15:'422c'})

# Make dataframe of each checklist
q_lst = [q_101,q_103E,q_102,q_103,q_104,q_105,

         q_301,q_301E,
         q_302,q_303,q_304,

         q_305A,q_306,q_307,q_308,
         q_309,q_310,q_311,q_312,q_313,

         q_401, q_402, q_403, q_404,q_405,q_407,q_409,
         q_410,q_411,q_412,q_413,q_414,q_415,q_416,q_417,
         q_418,q_419,q_420,q_421,q_422]

q_df = pd.concat(q_lst, axis=0, join='outer', ignore_index=False,
          keys=None, levels=None, names=None, verify_integrity=False,
          copy=False)

#%%%

# make feature of the month
q_df['month']  = pd.to_datetime(q_df['submitDate'], dayfirst=True).dt.month

#%%
'''Remove unnecesary features and early clean'''

q_df = q_df.drop(['ID','userID','submitTime','lang','completed','submitDate',
                  't','b','kode','floor','otherOfficeInterior','yearsinoffice',
                  'daysInOffice', 'otherSeating','otherConstellation','otherJob',

                  'gangarealer','kantine','skrivebord','noMask','kontor',

                  'commentLight','lightOtherDissatisfaction',

                  'airqualStuffy','airqualSmelly','commentAirqual',

                  'commentNoise','seating','officeHome','noiseOtherOutdoor',
                  'otherAirqualSource','noiseOther',

                  'commentIndoorClimate'],axis=1)

'''The binary features are given as 1 and nan. The nan features must,
    thus, be replaced by 0'''

q_bi_lst = ['photocopier', 'outside', 'printer',
            'perfume','food','tobaccoSmoke','cleaningProd','carpet',

            'noiseTraffic','noiseOverhearing','noiseOfficeEquipment',
            'noisePhoneTalk','noiseNeighborTalk','noisemechanical',
            'noiseTelephones',

            'lightTooDark','lightTooBright','lightTooLittleDaylight','lightTooMuchDaylight',
            'lightFlicker', 'lightReflections','lightColor',
            'lightColor', 'lightNoTask']

for column in q_bi_lst:
    q_df[column] = q_df[column].map({ 1:True,np.nan:False})

#%%%
''' Make DF only with people sat in meassured offices'''

q_office_df = q_df[q_df['pizza'].apply(lambda x: isinstance(x, (float,int, np.int64)))]

q_office_df = q_office_df[q_office_df['officeID'].notna()]

#%%%
''' Make sure only symptoms that disapear when occupant leaves the office
    are taken into account.'''

q_office_df= q_office_df.reset_index()

# The following refers to function in load_utils, that reduces the the four
# classes of each symptom into two, where 0 is no symptom in the last 4 weeks
# and 1 is the experience of the symptom.
q_office_df['symp0'] = q_office_df['symp0'].apply(lu.reduce_s)
q_office_df['symp1'] = q_office_df['symp1'].apply(lu.reduce_s)
q_office_df['symp2'] = q_office_df['symp2'].apply(lu.reduce_s)
q_office_df['symp3'] = q_office_df['symp3'].apply(lu.reduce_s)
q_office_df['symp4'] = q_office_df['symp4'].apply(lu.reduce_s)
q_office_df['symp5'] = q_office_df['symp5'].apply(lu.reduce_s)
q_office_df['symp6'] = q_office_df['symp6'].apply(lu.reduce_s)
q_office_df['symp7'] = q_office_df['symp7'].apply(lu.reduce_s)
q_office_df['symp8'] = q_office_df['symp8'].apply(lu.reduce_s)
q_office_df['symp9'] = q_office_df['symp9'].apply(lu.reduce_s)

# When an occupant experiences a symptom, they will either feel worse,
# better or no change after leaving the office. We are only interested
# in the office related symtoms.
#symp0:
q_office_df['betterorworse0'] = q_office_df['betterorworse0'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s0_better'] = ''
lu.sBetter(q_office_df,'symp0','betterorworse0','s0_better')

#symp1:
q_office_df['betterorworse1'] = q_office_df['betterorworse1'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s1_better'] = ''
lu.sBetter(q_office_df,'symp1','betterorworse1','s1_better')
#symp2:
q_office_df['betterorworse2'] = q_office_df['betterorworse2'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s2_better'] = ''
lu.sBetter(q_office_df,'symp2','betterorworse2','s2_better')

#symp3:
q_office_df['betterorworse3'] = q_office_df['betterorworse3'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s3_better'] = ''
lu.sBetter(q_office_df,'symp3','betterorworse1','s3_better')

#symp4:
q_office_df['betterorworse4'] = q_office_df['betterorworse4'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s4_better'] = ''
lu.sBetter(q_office_df,'symp4','betterorworse4','s4_better')

#symp5:
q_office_df['betterorworse5'] = q_office_df['betterorworse5'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s5_better'] = ''
lu.sBetter(q_office_df,'symp5','betterorworse5','s5_better')

#symp6:
q_office_df['betterorworse6'] = q_office_df['betterorworse6'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s6_better'] = ''
lu.sBetter(q_office_df,'symp6','betterorworse6','s6_better')

#symp7:
q_office_df['betterorworse7'] = q_office_df['betterorworse7'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s7_better'] = ''
lu.sBetter(q_office_df,'symp7','betterorworse7','s7_better')

#symp8:
q_office_df['betterorworse8'] = q_office_df['betterorworse8'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s8_better'] = ''
lu.sBetter(q_office_df,'symp8','betterorworse8','s8_better')

#symp9:
q_office_df['betterorworse9'] = q_office_df['betterorworse9'].map({"worse":0,
                                                                   "nochange":0,
                                                                   "better":1,
                                                                   np.nan: 0})
q_office_df['s9_better'] = ''
lu.sBetter(q_office_df,'symp9','betterorworse9','s9_better')

# Below the experience of at least one or two different office-related
# symptoms are made into a binary feature
lu.SBS_1(q_office_df,'s0_better','s1_better','s2_better','s3_better','s4_better','s5_better','s6_better','s7_better','s8_better','s9_better','SBS1')
lu.SBS_2(q_office_df,'s0_better','s1_better','s2_better','s3_better','s4_better','s5_better','s6_better','s7_better','s8_better','s9_better','SBS2')

#%%%
# The following was used to assess the distributions of each symptom, to see
# what symptoms to make single symptom models of.

# plot symptoms in SBS

# fig, axs = plt.subplots(5, 2, figsize=(20, 8))
# sns.catplot(data=q_office_df, x="s0", kind="count", palette="ch:.25",aspect=1).set(title='Dry or irritated eyes')
# sns.catplot(data=q_office_df, x="s1", kind="count", palette="ch:.25",aspect=1).set(title='Headache')
# sns.catplot(data=q_office_df, x="s2", kind="count", palette="ch:.25",aspect=1).set(title='Sore or dry throat')
# sns.catplot(data=q_office_df, x="s3", kind="count", palette="ch:.25",aspect=1).set(title='Tiredness or fatigue')
# sns.catplot(data=q_office_df, x="s4", kind="count", palette="ch:.25",aspect=1).set(title='Stuffy or runny nose')
# sns.catplot(data=q_office_df, x="s5", kind="count", palette="ch:.25",aspect=1).set(title='Cough')
# sns.catplot(data=q_office_df, x="s6", kind="count", palette="ch:.25",aspect=1).set(title='Sneezing')
# sns.catplot(data=q_office_df, x="s7", kind="count", palette="ch:.25",aspect=1).set(title='Difficult to concentrate')
# sns.catplot(data=q_office_df, x="s8", kind="count", palette="ch:.25",aspect=1).set(title='Shortness of breath')
# sns.catplot(data=q_office_df, x="s9", kind="count", palette="ch:.25",aspect=1).set(title='Dry skin')
# plt.show()

#symp0: Dry or irritated eyes - good
#symp0: Headache - good
#symp3: Tiredness or fatigue - good
#symp7: Difficult to concentrate - good

#symp2: Sore or dry throat - less than 30
#symp4: Stuffy or runny nose - less than 20
#symp5: cough - less than 20
#symp6: sneezing - less than 20
#symp8: Shortness of breath - less than 10
#symp9: dry skin - less than 20

print("s0",q_office_df['s0'].value_counts())
print("s1",q_office_df['s1'].value_counts())
print("s2",q_office_df['s2'].value_counts())
print("s3",q_office_df['s3'].value_counts())
print("s4",q_office_df['s4'].value_counts())
print("s5",q_office_df['s5'].value_counts())
print("s6",q_office_df['s6'].value_counts())
print("s7",q_office_df['s7'].value_counts())
print("s8",q_office_df['s8'].value_counts())
print("s9",q_office_df['s9'].value_counts())

'''0: Not in the past 4 weeks
   1: 1-3 days in the past 4 weeks
   2: 1-3 days per week in the past 4 weeks
   3: Every workday in the past 4 weeks
'''

#%%%

sns.catplot(data=q_office_df, x="SBS1", kind="count", palette="ch:.25",aspect=40/10)
print(q_office_df['SBS1'].value_counts())
#%%%

#plot distribution of at least two office-related symptoms
sns.catplot(data=q_office_df, x="SBS2", kind="count", palette="ch:.25",height=4,aspect=3/2).set(title='Multiple symptoms')
plt.show()
print(q_office_df['SBS2'].value_counts())

#%%%

#Drop features that are not used as targets

q_office_df = q_office_df.drop(['index',
                            'q0','q1','q2','q3','q4','q5','q6','q7','q8','q9','q10',
                            'q11','thisweek_0','thisweek_1','thisweek_2','thisweek_3',
                            'thisweek_4','thisweek_5','thisweek_6','thisweek_7',
                            'thisweek_8','thisweek_9','thisweek_10','thisweek_11',

                            'betterorworse0','symp0',
                            'betterorworse1','symp1',
                            'betterorworse3','symp3',
                            'betterorworse7','symp7',

                            's2_better','s4_better','s5_better',
                            's6_better','s8_better','s9_better',

                            'symp2','symp4','symp5',
                            'symp6','symp8','symp9',
                            'betterorworse2','betterorworse4',
                            'betterorworse5','betterorworse6',
                            'betterorworse8','betterorworse9',

                            's0','s1','s2','s3','s4','s5','s6','s7','s8','s9',

                            'performanceFactor0','performanceFactor1',
                            'performanceFactor2','performanceFactor3',
                            'performanceFactor4','performanceFactor5',
                            'performanceFactor6','performanceFactor7',
                            'performanceFactor8','performanceFactor9',
                            'performanceFactor10','pizza'],axis=1)
#%%%
#symp0: Dry or irritated eyes - good
#symp3: Tiredness or fatigue - good
#symp7: Difficult to concentrate - good

#symp2: Sore or dry throat - less than 30
#symp4: Stuffy or runny nose - less than 20
#symp5: cough - less than 20
#symp6: sneezing - less than 20
#symp8: Shortness of breath - less than 10
#symp9: dry skin - less than 20

# sns.catplot(data=q_office_df, x="symp7", kind="count", palette="ch:.25",aspect=40/10)
# sns.catplot(data=q_office_df, x="betterorworse7", kind="count", palette="ch:.25",aspect=40/10)

#%% - Delete unwanted dataframes

del [q_101,q_102,q_103,q_104,q_105,q_103E,

     q_301,q_302,q_303,q_304,

     q_301E,

     q_305A,q_306,q_307,q_308,
     q_309,q_310,q_311,q_312,q_313,

     q_401, q_402, q_403, q_404,q_405,q_407,q_409,
     q_410,q_411,q_412,q_413,q_414,q_415,q_416,q_417,q_418,q_419,
     q_420, q_421, q_422,
     q_lst, q_bi_lst,column]

# %%

q_office_df.to_csv('../temp_data/q_office_df.csv', index=False)


import collections
import hashlib
import time
import multiprocessing as mp

import torch
import numpy as np
import torch.nn as nn
import scipy.stats
import matplotlib.pyplot as plt
from PIL import Image

import jax
import jax.numpy as jn
import objax
import scipy.optimize
import numpy as np
import multiprocessing as mp

# Objax neural network that's going to embed patches to a
# low dimensional space to guess if two patches correspond
# to the same orginal image.
class Model(objax.Module):
    def __init__(self):
        IN = 15
        H = 64
        self.encoder =objax.nn.Sequential([
            objax.nn.Linear(IN, H),
            objax.functional.leaky_relu,
            objax.nn.Linear(H, H),
            objax.functional.leaky_relu,
            objax.nn.Linear(H, 8)])
        self.decoder =objax.nn.Sequential([
            objax.nn.Linear(IN, H),
            objax.functional.leaky_relu,
            objax.nn.Linear(H, H),
            objax.functional.leaky_relu,
            objax.nn.Linear(H, 8)])
        self.scale = objax.nn.Linear(1, 1, use_bias=False)
    def encode(self, x):
        # Encode turns original images into feature space
        a = self.encoder(x)
        a = a/jn.sum(a**2,axis=-1,keepdims=True)**.5
        return a
    def decode(self, x):
        # And decode turns encoded images into feature space
        a = self.decoder(x)
        a = a/jn.sum(a**2,axis=-1,keepdims=True)**.5
        return a

# Proxy dataset for analysis
class ImageNet:
    num_chan = 3
    private_kernel_size = 16
    hidden_dim = 2048
    img_size = (256, 256)
    private_depth = 7
    def __init__(self, remove):
        self.remove_pixel_shuffle = remove

# Original dataset as used in the NeuraCrypt paper
class Xray:
    num_chan = 1
    private_kernel_size = 16
    hidden_dim = 2048
    img_size = (256, 256)
    private_depth = 4
    def __init__(self, remove):
        self.remove_pixel_shuffle = remove

## The following class is taken directly from the NeuraCrypt codebase.
## https://github.com/yala/NeuraCrypt
## which is originally licensed under the MIT License
class PrivateEncoder(nn.Module):
    def __init__(self, args, width_factor=1):
        super(PrivateEncoder, self).__init__()
        self.args = args
        input_dim = args.num_chan
        patch_size = args.private_kernel_size
        output_dim = args.hidden_dim
        num_patches =  (args.img_size[0] // patch_size) **2
        self.noise_size = 1

        args.input_dim = args.hidden_dim

        layers  = [
                    nn.Conv2d(input_dim, output_dim * width_factor, kernel_size=patch_size, dilation=1 ,stride=patch_size),
                    nn.ReLU()
                    ]
        for _ in range(self.args.private_depth):
            layers.extend( [
                nn.Conv2d(output_dim * width_factor, output_dim * width_factor , kernel_size=1, dilation=1, stride=1),
                nn.BatchNorm2d(output_dim * width_factor, track_running_stats=False),
                nn.ReLU()
            ])

        self.image_encoder = nn.Sequential(*layers)

        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches, output_dim * width_factor))

        self.mixer = nn.Sequential( *[
            nn.ReLU(),
            nn.Linear(output_dim * width_factor, output_dim)
            ])

    def forward(self, x):
        encoded = self.image_encoder(x)
        B, C, H,W = encoded.size()
        encoded = encoded.view([B, -1, H*W]).transpose(1,2)
        encoded += self.pos_embedding
        encoded  = self.mixer(encoded)

        ## Shuffle indicies
        if not self.args.remove_pixel_shuffle:
            shuffled = torch.zeros_like(encoded)
            for i in range(B):
                idx = torch.randperm(H*W, device=encoded.device)
                for j, k in enumerate(idx):
                    shuffled[i,j] = encoded[i,k]
            encoded = shuffled

        return encoded
## End copied code

def setup(ds):
    """
    Load the datasets to use. Nothing interesting to see.
    """
    global x_train, y_train
    if ds == 'imagenet':
        import torchvision
        transform = torchvision.transforms.Compose([
            torchvision.transforms.Resize(256),
            torchvision.transforms.CenterCrop(256),
            torchvision.transforms.ToTensor()])
        imagenet_data = torchvision.datasets.ImageNet('/mnt/data/datasets/unpacked_imagenet_pytorch/',
                                                      split='val',
                                                      transform=transform)
        data_loader = torch.utils.data.DataLoader(imagenet_data,
                                                  batch_size=100,
                                                  shuffle=True,
                                                  num_workers=8)
        r = []
        for x,_ in data_loader:
            if len(r) > 1000: break
            print(x.shape)
            r.extend(x.numpy())
        x_train = np.array(r)
        print(x_train.shape)
    elif ds == 'xray':
        import torchvision
        transform = torchvision.transforms.Compose([
            torchvision.transforms.Resize(256),
            torchvision.transforms.CenterCrop(256),
            torchvision.transforms.ToTensor()])
        imagenet_data = torchvision.datasets.ImageFolder('CheXpert-v1.0/train',
                                                      transform=transform)
        data_loader = torch.utils.data.DataLoader(imagenet_data,
                                                  batch_size=100,
                                                  shuffle=True,
                                                  num_workers=8)
        r = []
        for x,_ in data_loader:
            if len(r) > 1000: break
            print(x.shape)
            r.extend(x.numpy())
        x_train = np.array(r)
        print(x_train.shape)
    elif ds == 'challenge':
        x_train = np.load("orig-7.npy")
        print(np.min(x_train), np.max(x_train), x_train.shape)
    else:
        raise

def gen_train_data():
    """
    Generate aligned training data to train a patch similarity function.
    Given some original images, generate lots of encoded versions.
    """
    global encoded_train, original_train

    encoded_train = []
    original_train = []

    args = Xray(True)

    C = 100
    for i in range(30):
        print(i)
        torch.manual_seed(int(time.time()))
        e = PrivateEncoder(args).cuda()
        batch = np.random.randint(0, len(x_train), size=C)
        xin = x_train[batch]

        r = []
        for i in range(0,C,32):
            r.extend(e(torch.tensor(xin[i:i+32]).cuda()).detach().cpu().numpy())
        r = np.array(r)

        encoded_train.append(r)
        original_train.append(xin)

def features_(x, moments=15, encoded=False):
    """
    Compute higher-order moments for patches in an image to use as
    features for the neural network.
    """
    x = np.array(x, dtype=np.float32)
    dim = 2
    arr = np.array([np.mean(x, dim)] + [abs(scipy.stats.moment(x, moment=i, axis=dim))**(1/i) for i in range(1,moments)])

    return arr.transpose((1,2,0))

def features(x, encoded):
    """
    Given the original images or the encoded images, generate the
    features to use for the patch similarity function.
    """
    print('start shape',x.shape)
    if len(x.shape) == 3:
        x = x - np.mean(x,axis=0,keepdims=True)
    else:
        # count x 100 x 256 x 768
        print(x[0].shape)
        x = x - np.mean(x,axis=1,keepdims=True)
        # remove per-neural-network dimension
        x = x.reshape((x.shape[0] * x.shape[1],) + x.shape[2:])
    p = mp.Pool(96)
    B = len(x) // 96
    print(1)
    bs = [x[i:i+B] for i in range(0,len(x),B)]
    print(2)
    r = p.map(features_, bs)
    #r = features_(bs[0][:100])
    print(3)
    p.close()
    #r = np.array(r)
    #print('finish',r.shape)
    return np.concatenate(r, axis=0)

def get_train_features():
    """
    Create features for the entire datasets.
    """
    global xs_train, ys_train
    print(x_train.shape)
    original_train_ = np.array(original_train)
    encoded_train_ = np.array(encoded_train)

    print("Computing features")
    ys_train = features(encoded_train_, True)

    patch_size = 16
    ss = original_train_.shape[3] // patch_size
    # Okay so this is an ugly transpose block.
    # We are going from [outer_batch, batch_size, channels, width, height
    # to [outer_batch, batch_size, channels, width/patch_size, patch_size, height/patch_size, patch_size]
    # Then we reshape this and flatten so that we end up with
    # [other_batch, batch_size, width/patch_size, height_patch_size, patch_size**2*channels]
    # So that now we can run features on the last dimension
    original_train_ = original_train_.reshape((original_train_.shape[0],
                                               original_train_.shape[1],
                                               original_train_.shape[2],
                                               ss,patch_size,ss,patch_size)).transpose((0,1,3,5,2,4,6)).reshape((original_train_.shape[0], original_train_.shape[1], ss**2, patch_size**2))

    xs_train = features(original_train_, False)

    print(xs_train.shape, ys_train.shape)

def train_model():
    """
    Train the patch similarity function
    """
    global ema, model

    model = Model()
    def loss(x, y):
        """
        K-way contrastive loss as in SimCLR et al.
        The idea is that we should embed x and y so that they are similar
        to each other, and dis-similar from others. To do this we have a
        softmx loss over one dimension to make the values large on the diagonal
        and small off-diagonal.
        """
        a = model.encode(x)
        b = model.decode(y)

        mat = a@b.T
        return objax.functional.loss.cross_entropy_logits_sparse(
            logits=jn.exp(jn.clip(model.scale.w.value, -2, 4)) * mat,
            labels=np.arange(a.shape[0])).mean()

    ema = objax.optimizer.ExponentialMovingAverage(model.vars(), momentum=0.999)
    gv = objax.GradValues(loss, model.vars())

    encode_ema = ema.replace_vars(lambda x: model.encode(x))
    decode_ema = ema.replace_vars(lambda y: model.decode(y))

    def train_op(x, y):
        """
        No one was ever fired for using Adam with 1e-4.
        """
        g, v = gv(x, y)
        opt(1e-4, g)
        ema()
        return v

    opt = objax.optimizer.Adam(model.vars())
    train_op = objax.Jit(train_op, gv.vars() + opt.vars() + ema.vars())

    ys_ = ys_train

    print(ys_.shape)

    xs_ = xs_train.reshape((-1, xs_train.shape[-1]))
    ys_ = ys_.reshape((-1, ys_train.shape[-1]))

    # The model scale trick here is taken from CLIP.
    # Let the model decide how confident to make its own predictions.
    model.scale.w.assign(jn.zeros((1,1)))

    valid_size = 1000

    print(xs_train.shape)
    # SimCLR likes big batches
    B = 4096
    for it in range(80):
        print()
        ms = []
        for i in range(1000):
            # First batch is smaller, to make training more stable
            bs = [B // 64, B][it>0]
            batch = np.random.randint(0, len(xs_)-valid_size, size=bs)
            r = train_op(xs_[batch], ys_[batch])

            # This shouldn't happen, but if it does, better to bort early
            if np.isnan(r):
                print("Die on nan")
                print(ms[-100:])
                return
            ms.append(r)

        print('mean',np.mean(ms), 'scale', model.scale.w.value)
        print('loss',loss(xs_[-100:], ys_[-100:]))

        a = encode_ema(xs_[-valid_size:])
        b = decode_ema(ys_[-valid_size:])

        br = b[np.random.permutation(len(b))]

        print('score',np.mean(np.sum(a*b,axis=(1)) - np.sum(a*br,axis=(1))),
              np.mean(np.sum(a*b,axis=(1)) > np.sum(a*br,axis=(1))))
    ckpt = objax.io.Checkpoint("saved", keep_ckpts=0)
    ema.replace_vars(lambda: ckpt.save(model.vars(), 0))()

def load_challenge():
    """
    Load the challenge datast for attacking
    """
    global xs, ys, encoded, original, ooriginal
    print("SETUP: Loading matrixes")
    # The encoded images
    encoded = np.load("challenge-7.npy")
    # And the original images
    ooriginal = original = np.load("orig-7.npy")

    print("Sizes", encoded.shape, ooriginal.shape)

    # Again do that ugly resize thing to make the features be on the last dimension
    # Look up above to see what's going on.
    patch_size = 16
    ss = original.shape[2] // patch_size
    original = ooriginal.reshape((original.shape[0],1,ss,patch_size,ss,patch_size))
    original = original.transpose((0,2,4,1,3,5))
    original = original.reshape((original.shape[0], ss**2, patch_size**2))

def match_sub(args):
    """
    Find the best way to undo the permutation between two images.
    """
    vec1, vec2 = args
    value = np.sum((vec1[None,:,:] - vec2[:,None,:])**2,axis=2)
    row, col = scipy.optimize.linear_sum_assignment(value)
    return col

def recover_local_permutation():
    """
    Given a set of encoded images, return a new encoding without permutations
    """
    global encoded, ys

    p = mp.Pool(96)
    print('recover local')
    local_perm = p.map(match_sub, [(encoded[0], e) for e in encoded])
    local_perm = np.array(local_perm)

    encoded_perm = []

    for i in range(len(encoded)):
        encoded_perm.append(encoded[i][np.argsort(local_perm[i])])

    encoded_perm = np.array(encoded_perm)

    encoded = np.array(encoded_perm)

    p.close()

def recover_better_local_permutation():
    """
    Given a set of encoded images, return a new encoding, but better!
    """
    global encoded, ys

    # Now instead of pairing all images to image 0, we compute the mean l2 vector
    # and then pair all images onto the mean vector. Slightly more noise resistant.
    p = mp.Pool(96)
    target = encoded.mean(0)
    local_perm = p.map(match_sub, [(target, e) for e in encoded])
    local_perm = np.array(local_perm)

    # Probably we didn't change by much, generally <0.1%
    print('improved changed by', np.mean(local_perm != np.arange(local_perm.shape[1])))

    encoded_perm = []

    for i in range(len(encoded)):
        encoded_perm.append(encoded[i][np.argsort(local_perm[i])])

    encoded = np.array(encoded_perm)

    p.close()

def compute_patch_similarity():
    """
    Compute the feature vectors for each patch using the trained neural network.
    """
    global xs, ys, xs_image, ys_image

    print("Computing features")
    ys = features(encoded, encoded=True)
    xs = features(original, encoded=False)

    model = Model()
    ckpt = objax.io.Checkpoint("saved", keep_ckpts=0)
    ckpt.restore(model.vars())

    xs_image = model.encode(xs)
    ys_image = model.decode(ys)
    assert xs.shape[0] == xs_image.shape[0]
    print("Done")

def match(args, ret_col=False):
    """
    Compute the similarity between image features and encoded features.
    """
    vec1, vec2s = args
    r = []
    open("/tmp/start%d.%d"%(np.random.randint(10000),time.time()),"w").write("hi")
    for vec2 in vec2s:
        value = np.sum(vec1[None,:,:] * vec2[:,None,:],axis=2)

        row, col = scipy.optimize.linear_sum_assignment(-value)
        r.append(value[row,col].mean())
    return r

def recover_global_matching_first():
    """
    Recover the global matching of original to encoded images by doing
    an all-pairs matching problem
    """
    global global_matching, ys_image, encoded

    matrix = []
    p = mp.Pool(96)
    xs_image_ = np.array(xs_image)
    ys_image_ = np.array(ys_image)

    matrix = p.map(match, [(x, ys_image_) for x in xs_image_])
    matrix = np.array(matrix).reshape((xs_image.shape[0],
                                       xs_image.shape[0]))

    row, col = scipy.optimize.linear_sum_assignment(-np.array(matrix))
    global_matching = np.argsort(col)
    print('glob',list(global_matching))

    p.close()

def recover_global_permutation():
    """
    Find the way that the encoded images are permuted off of the original images
    """
    global global_permutation

    print("Glob match", global_matching)
    overall = []
    for i,j in enumerate(global_matching):
        overall.append(np.sum(xs_image[j][None,:,:] * ys_image[i][:,None,:],axis=2))

    overall = np.mean(overall, 0)

    row, col = scipy.optimize.linear_sum_assignment(-overall)

    try:
        print("Changed frac:", np.mean(global_permutation!=np.argsort(col)))
    except:
        pass

    global_permutation = np.argsort(col)

def recover_global_matching_second():
    """
    Match each encoded image with its original encoded image,
    but better by relying on the global permutation.
    """
    global global_matching_second, global_matching

    ys_fix = []
    for i in range(ys_image.shape[0]):
        ys_fix.append(ys_image[i][global_permutation])
    ys_fix = np.array(ys_fix)

    print(xs_image.shape)

    sims = []
    for i in range(0,len(xs_image),10):
        tmp = np.mean(xs_image[None,:,:,:] * ys_fix[i:i+10][:,None,:,:],axis=(2,3))
        sims.extend(tmp)
    sims = np.array(sims)
    print(sims.shape)

    row, col = scipy.optimize.linear_sum_assignment(-sims)

    print('arg',sims.argmax(1))

    print("Same matching frac", np.mean(col == global_matching) )
    print(col)
    global_matching = col

def extract_by_training(resume):
    """
    Final recovery process by extracting the neural network
    """
    global inverse

    device = torch.device('cuda:1')

    if not resume:
        inverse = PrivateEncoder(Xray(True)).cuda(device)

    # More adam to train.
    optimizer = torch.optim.Adam(inverse.parameters(), lr=0.0001)

    this_xs = ooriginal[global_matching]
    this_ys = encoded[:,global_permutation,:]

    for i in range(2000):
        idx = np.random.random_integers(0, len(this_xs)-1, 32)
        xbatch = torch.tensor(this_xs[idx]).cuda(device)
        ybatch = torch.tensor(this_ys[idx]).cuda(device)

        optimizer.zero_grad()

        guess_output = inverse(xbatch)
        # L1 loss because we don't want to be sensitive to outliers
        error = torch.mean(torch.abs(guess_output-ybatch))
        error.backward()

        optimizer.step()

        print(error)

def test_extract():
    """
    Now we can recover the matching much better by computing the estimated
    encodings for each original image.
    """
    global err, global_matching, guessed_encoded, smatrix

    device = torch.device('cuda:1')

    print(ooriginal.shape, encoded.shape)

    out = []
    for i in range(0,len(ooriginal),32):
        print(i)
        out.extend(inverse(torch.tensor(ooriginal[i:i+32]).cuda(device)).cpu().detach().numpy())

    guessed_encoded = np.array(out)

    # Now we have to compare each encoded image with every other original image.
    # Do this fast with some matrix multiplies.

    out = guessed_encoded.reshape((len(encoded), -1))
    real = encoded[:,global_permutation,:].reshape((len(encoded), -1))
    @jax.jit
    def foo(x, y):
        return jn.square(x[:,None] - y[None,:]).sum(2)

    smatrix = np.zeros((len(out), len(out)))

    B = 500
    for i in range(0,len(out),B):
        print(i)
        for j in range(0,len(out),B):
            smatrix[i:i+B, j:j+B] = foo(out[i:i+B], real[j:j+B])

    # And the final time you'l have to look at a min weight matching, I promise.
    row, col = scipy.optimize.linear_sum_assignment(np.array(smatrix))
    r = np.array(smatrix)

    print(list(row)[::100])

    print("Differences", np.mean(np.argsort(col) != global_matching))

    global_matching = np.argsort(col)

def perf(steps=[]):
    if len(steps) == 0:
        steps.append(time.time())
    else:
        print("Last Time Elapsed:", time.time()-steps[-1], ' Total Time Elapsed:', time.time()-steps[0])
        steps.append(time.time())
    time.sleep(1)

if __name__ == "__main__":
    if True:
        perf()
        setup('challenge')
        perf()
        gen_train_data()
        perf()
        get_train_features()
        perf()
        train_model()
        perf()

    if True:
        load_challenge()
        perf()
        recover_local_permutation()
        perf()
        recover_better_local_permutation()
        perf()
        compute_patch_similarity()
        perf()
        recover_global_matching_first()
        perf()

    for _ in range(3):
        recover_global_permutation()
        perf()
        recover_global_matching_second()
        perf()

    for i in range(3):
        recover_global_permutation()
        perf()
        extract_by_training(i > 0)
        perf()
        test_extract()
        perf()
    print(perf())


from page_object.ChooserType import ChooserType
import sys
'''
Проверяет все цены каждого типа/размера/ткани.
Если цена не соответствует РРЦ, пишет в папку log всю инфу с сортировкой по магазину
Скрипт ориентируется на большую цену в карточке товара,
поэтому лучше не проверять в первые дни после изменения цены/акции,
чтобы все магазины успели загрузить выгрузку.
'''

def test_2(browser):
    try:
        all_matras_link = list(map(str.rstrip, open('all_links.txt').readlines()))
    except:
        print('Сначала запусти test_1 для парсинга ссылок')
        sys.exit()

    matras = ChooserType(browser)
    for link in all_matras_link:
        matras.get_card_page(link)
        if matras.is_demp():  # если есть ли демпинг, сравниваем все цены и пишем в лог
            matras.get_all_price()


import os
import unittest
from datetime import datetime, timedelta

import numpy as np

from karabo.imaging.imager import Imager
from karabo.simulation.beam import BeamPattern
from karabo.simulation.interferometer import InterferometerSimulation
from karabo.simulation.observation import ObservationLong
from karabo.simulation.sky_model import SkyModel
from karabo.simulation.telescope import Telescope
from karabo.simulation.visibility import Visibility
from karabo.test import data_path

class MyTestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        # make dir for result files
        if not os.path.exists("result/"):
            os.makedirs("result/")

    def test_fit_element(self):
        tel = Telescope.get_MEERKAT_Telescope()
        beam = BeamPattern(f"{data_path}/run5.cst")
        beam.fit_elements(tel, freq_hz=1.0e08, avg_frac_error=0.5)

    def test_katbeam(self):
        beampixels = BeamPattern.get_meerkat_uhfbeam(
            f=800, pol="I", beamextentx=40, beamextenty=40
        )
        BeamPattern.show_kat_beam(
            beampixels[0], 40, 800, "I", path="./result/katbeam_beam.png"
        )

    def test_eidosbeam(self):
        npix = 500
        dia = 10
        thres = 0
        ch = 0
        B_ah = BeamPattern.get_eidos_holographic_beam(
            npix=npix, ch=ch, dia=dia, thres=thres, mode="AH"
        )
        BeamPattern.show_eidos_beam(B_ah, path="./result/eidos_AH_beam.png")
        B_em = BeamPattern.get_eidos_holographic_beam(
            npix=npix, ch=ch, dia=dia, thres=thres, mode="EM"
        )
        BeamPattern.show_eidos_beam(B_em, path="./result/eidos_EM_beam.png")
        BeamPattern.eidos_lineplot(
            B_ah, B_em, npix, path="./result/eidos_residual_beam.png"
        )

    def test_long_observations(self):
        # skips `input` during unit tests if using `karabo.util.data_util.input_wrapper`
        os.environ["SKIP_INPUT"] = str(True)
        number_of_days = 3
        hours_per_day = 4
        enable_array_beam = False
        vis_path = "./karabo/test/data"
        combined_vis_filepath = os.path.join(vis_path, "combined_vis.ms")
        xcstfile_path = os.path.join(vis_path, "cst_like_beam_port_1.txt")
        ycstfile_path = os.path.join(vis_path, "cst_like_beam_port_2.txt")
        sky = SkyModel()
        sky_data = np.array(
            [
                [20.0, -30.0, 1, 0, 0, 0, 100.0e6, -0.7, 0.0, 0, 0, 0],
                [20.0, -30.5, 3, 2, 2, 0, 100.0e6, -0.7, 0.0, 0, 0, 45],
                [20.5, -30.5, 3, 0, 0, 2, 100.0e6, -0.7, 0.0, 0, 0, -10],
            ]
        )
        sky.add_point_sources(sky_data)
        telescope = Telescope.get_MEERKAT_Telescope()
        observation_long = ObservationLong(
            mode="Tracking",
            phase_centre_ra_deg=20.0,
            start_date_and_time=datetime(2000, 1, 1, 11, 00, 00, 521489),
            length=timedelta(hours=hours_per_day, minutes=0, seconds=0, milliseconds=0),
            phase_centre_dec_deg=-30.0,
            number_of_time_steps=7,
            start_frequency_hz=1.0e9,
            frequency_increment_hz=1e6,
            number_of_channels=3,
            number_of_days=number_of_days,
        )
        beam_polX = BeamPattern(
            cst_file_path=xcstfile_path,
            telescope=telescope,
            freq_hz=observation_long.start_frequency_hz,
            pol="X",
            avg_frac_error=0.001,
            beam_method="Gaussian Beam",
        )
        beam_polY = BeamPattern(
            cst_file_path=ycstfile_path,
            telescope=telescope,
            freq_hz=observation_long.start_frequency_hz,
            pol="Y",
            avg_frac_error=0.001,
            beam_method="Gaussian Beam",
        )
        simulation = InterferometerSimulation(
            vis_path=vis_path,
            channel_bandwidth_hz=2e7,
            time_average_sec=7,
            noise_enable=False,
            noise_seed="time",
            noise_freq="Range",
            noise_rms="Range",
            noise_start_freq=1.0e9,
            noise_inc_freq=1.0e6,
            noise_number_freq=1,
            noise_rms_start=0.1,
            noise_rms_end=1,
            enable_numerical_beam=enable_array_beam,
            enable_array_beam=enable_array_beam,
            beam_polX=beam_polX,
            beam_polY=beam_polY,
        )
        # -------- Iterate over days
        visiblity_files = simulation.run_simulation(
            telescope=telescope,
            sky=sky,
            observation=observation_long,
        )

        # visibility.write_to_file("/home/rohit/karabo/karabo-pipeline/karabo/test/result/beam/beam_vis.ms")
        # ---------- Combine the Visibilties --------------
        # visiblity_files= [
        #     './karabo/test/data/beam_vis_1.vis',
        #     './karabo/test/data/beam_vis_2.vis',
        #     './karabo/test/data/beam_vis_3.vis',
        # ]
        Visibility.combine_vis(
            number_of_days, visiblity_files, combined_vis_filepath, day_comb=False
        )
        visibilties = Visibility.read_from_file(combined_vis_filepath)
        # imaging cellsize is over-written in the Imager based on max uv dist.
        imager = Imager(visibilties, imaging_npixel=4096, imaging_cellsize=1.0e-5)
        imager.get_dirty_image()
        # dirty.write_to_file("./test/result/beam/beam_vis.fits",overwrite=True)
        # dirty.plot(colobar_label="Flux Density (Jy)", filename="combine_vis.png")

if __name__ == "__main__":
    unittest.main()


import requests
from datetime import datetime, timedelta
from flask import Flask, render_template, request
import os

app = Flask(__name__)

current_datetime = datetime.now()
today_date = current_datetime.strftime('%d-%m-%Y')

def format_date(date):
    formatted_date = date.strftime("%d %b %Y")
    internal_date = date.strftime("%Y%m%d")
    return formatted_date, internal_date

def generate_dates():
    today = datetime.today()
    future_dates = []
    past_dates = []

    for i in range(3, 0, -1):
        past_date = today - timedelta(days=i)
        past_dates.append(format_date(past_date))

    todays_date = format_date(today)
    return past_dates + [todays_date]

def get_data(user_input, date_input):
    global travel_date

    train_number = user_input
    api_key = os.environ["train_key"]
    date_obj = datetime.strptime(date_input, "%Y%m%d")
    selected_date = date_obj.strftime("%d %b %Y")
    if selected_date[0] == "0":
        selected_date = selected_date[1:]
    url = f"https://www.trainman.in/services/get-ntes-running-status/{train_number}?key=" \
          f"{api_key}&int=1&refresh=true&date={selected_date}"

    headers = {'User-Agent': 'Chrome/91.0.4472.114'}
    response = requests.get(url=url, headers=headers)
    Data = response.json()

    current_rake_station = None
    rakes = Data['rakes']

    for Rakes in rakes:
        if Rakes['startDate'] == selected_date:
            current_rake_station = Rakes['stations']

    station_with_halt = []
    stations_data = []
    for n in range(len(current_rake_station)):
        if current_rake_station[n]['halt'] != "00:00":
            station_with_halt.append(current_rake_station[n])

    station_with_halt.insert(0, current_rake_station[0])
    station_with_halt[0]['arrive'] = 'Start'
    station_with_halt.append(current_rake_station[-1])
    station_with_halt[-1]['depart'] = 'End'

    for n in range(len(station_with_halt)):
        station_name = station_with_halt[n].get('sname', None)
        station_code = station_with_halt[n].get('stnCode', None)
        station_day = station_with_halt[n].get('day', None)
        station_arrival = station_with_halt[n].get('arrive', None)
        station_dept = station_with_halt[n].get('depart', None)
        station_delay = station_with_halt[n].get('delayDep', None)
        station_departed_bool = station_with_halt[n].get('dep', None)
        station_arrival_bool = station_with_halt[n].get('arr', None)

        DELAY_D = station_with_halt[n].get('delayDep', None)
        if DELAY_D is not None:
            if DELAY_D >= 60:
                hours = DELAY_D // 60
                minutes = DELAY_D % 60
                DELAY_D = f"{hours} hr {minutes} min"
            else:
                DELAY_D = f"{DELAY_D} min"

        if station_delay is not None and station_delay > 0:
            station_status = f"Late by: {DELAY_D}"
        elif station_delay is not None and station_delay == 0:
            station_status = "On time"
        else:
            DELAY_d = abs(int(DELAY_D[:3]))
            station_status = f"Early by: {DELAY_d} min"

        station_info = {'station_name': station_name, 'station_code': station_code, 'station_day': station_day,
                        'station_arrival': station_arrival, 'station_dept': station_dept, 'station_delay':
                            station_delay, 'DELAY_D': DELAY_D, 'station_status': station_status,
                        'station_departed_bool': station_departed_bool, 'station_arrival_bool': station_arrival_bool}

        stations_data.append(station_info)

    return stations_data

@app.route('/')
def index():
    display_dates = generate_dates()
    return render_template('home.html', available_dates=display_dates)

@app.route('/process', methods=['POST'])
def process():
    user_input = request.form['user_input']
    Date_input = request.form['date_input']
    formatted_date = datetime.strptime(Date_input, "%Y%m%d")
    start_date = formatted_date.strftime("%d %b %Y")

    stations_data = get_data(user_input, Date_input)
    image_links = []
    crossed = 0

    for n, station in enumerate(stations_data):
        if station['station_departed_bool']:
            image_links.append("/static/leave.png")
            crossed = n
        elif station['station_arrival_bool'] and station['station_dept'] == 'End':
            image_links.append("/static/leave.png")
        elif not station['station_departed_bool'] and not station['station_arrival_bool']:
            image_links.append("/static/upcoming.png")
        else:
            image_links.append("/static/dot.gif")

    if today_date < Date_input:
        for n, station in enumerate(stations_data):
            if n <= crossed:
                image_links[n] = "/static/leave.png"
            if n > crossed:
                image_links[n] = "/static/dot.gif"
                break

    return render_template("table.html", stations_data=stations_data, image_links=image_links, user_input=user_input,
                           start_date=start_date)

if __name__ == '__main__':
    app.run(debug=True)


def solution(s):
    answer = True
    numP = 0
    numY = 0
    for i in range(len(s)):
        if s[i]=='p' or s[i] == 'P':
            numP += 1
        elif s[i]=='y' or s[i] == 'Y':
            numY += 1

    if(numP!=numY):
        answer = False

    return answer

11.문자열내p와y의개수.py


import requests
import json

with open('token.json', 'r') as file:
    data = json.load(file)
    TOKEN = data[0]['token']

class Group:
    def __init__(self, id):
        self.id = id

    def get_params(self):
        return {
            'group_id': self.id,
            'fields': 'members_count',
            'access_token': TOKEN,
            'v': 5.92
        }

    def get_group_data(self):
        params = self.get_params()
        response_get_group = requests.get(
            'https://api.vk.com/method/groups.getById',
            params
        )
        group_data_dict = response_get_group.json()['response'][0]
        output_dict = {'name': group_data_dict['name'],
                       'gid': group_data_dict['id'],
                       'members_count': group_data_dict['members_count']}
        return output_dict


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 30 14:02:04 2017

@author: michael
"""

from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import numpy as np
import tensorflow as tf
from deepchem.nn import activations
from deepchem.nn import initializations
from deepchem.nn import model_ops

from deepchem.models.tensorgraph.layers import Layer
from deepchem.models.tensorgraph.layers import convert_to_layers

class Combine_AP(Layer):

  def __init__(self, **kwargs):
    super(Combine_AP, self).__init__(**kwargs)

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)
    A = in_layers[0].out_tensor
    P = in_layers[1].out_tensor
    out_tensor = [A, P]
    if set_tensors:
      self.out_tensor = out_tensor
    return out_tensor

class Separate_AP(Layer):

  def __init__(self, **kwargs):
    super(Separate_AP, self).__init__(**kwargs)

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)
    out_tensor = in_layers[0].out_tensor[0]
    if set_tensors:
      self.out_tensor = out_tensor
    return out_tensor

class WeaveLayer(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.WeaveLayer

  """

  def __init__(self,
               n_atom_input_feat=75,
               n_pair_input_feat=14,
               n_atom_output_feat=50,
               n_pair_output_feat=50,
               n_hidden_AA=50,
               n_hidden_PA=50,
               n_hidden_AP=50,
               n_hidden_PP=50,
               update_pair=True,
               init='glorot_uniform',
               activation='relu',
               dropout=None,
               **kwargs):
    """
    Parameters
    ----------
    n_atom_input_feat: int
      Number of features for each atom in input.
    n_pair_input_feat: int
      Number of features for each pair of atoms in input.
    n_atom_output_feat: int
      Number of features for each atom in output.
    n_pair_output_feat: int
      Number of features for each pair of atoms in output.
    n_hidden_XX: int
      Number of units(convolution depths) in corresponding hidden layer
    update_pair: bool, optional
      Whether to calculate for pair features,
      could be turned off for last layer
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    dropout: float, optional
      Dropout probability, not supported here

    """
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.update_pair = update_pair  # last weave layer does not need to update
    self.n_hidden_AA = n_hidden_AA
    self.n_hidden_PA = n_hidden_PA
    self.n_hidden_AP = n_hidden_AP
    self.n_hidden_PP = n_hidden_PP
    self.n_hidden_A = n_hidden_AA + n_hidden_PA
    self.n_hidden_P = n_hidden_AP + n_hidden_PP

    self.n_atom_input_feat = n_atom_input_feat
    self.n_pair_input_feat = n_pair_input_feat
    self.n_atom_output_feat = n_atom_output_feat
    self.n_pair_output_feat = n_pair_output_feat
    super(WeaveLayer, self).__init__(**kwargs)

  def build(self):
    """ Construct internal trainable weights.

    TODO(rbharath): Need to make this not set instance variables to
    follow style in other layers.
    """

    self.W_AA = self.init([self.n_atom_input_feat, self.n_hidden_AA])
    self.b_AA = model_ops.zeros(shape=[
        self.n_hidden_AA,
    ])

    self.W_PA = self.init([self.n_pair_input_feat, self.n_hidden_PA])
    self.b_PA = model_ops.zeros(shape=[
        self.n_hidden_PA,
    ])

    self.W_A = self.init([self.n_hidden_A, self.n_atom_output_feat])
    self.b_A = model_ops.zeros(shape=[
        self.n_atom_output_feat,
    ])

    self.trainable_weights = [
        self.W_AA, self.b_AA, self.W_PA, self.b_PA, self.W_A, self.b_A
    ]
    if self.update_pair:
      self.W_AP = self.init([self.n_atom_input_feat * 2, self.n_hidden_AP])
      self.b_AP = model_ops.zeros(shape=[
          self.n_hidden_AP,
      ])

      self.W_PP = self.init([self.n_pair_input_feat, self.n_hidden_PP])
      self.b_PP = model_ops.zeros(shape=[
          self.n_hidden_PP,
      ])

      self.W_P = self.init([self.n_hidden_P, self.n_pair_output_feat])
      self.b_P = model_ops.zeros(shape=[
          self.n_pair_output_feat,
      ])

      self.trainable_weights.extend(
          [self.W_AP, self.b_AP, self.W_PP, self.b_PP, self.W_P, self.b_P])

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """ description and explanation refer to deepchem.nn.WeaveLayer
    parent layers: [atom_features, pair_features], pair_split, atom_to_pair
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()

    atom_features = in_layers[0].out_tensor[0]
    pair_features = in_layers[0].out_tensor[1]

    pair_split = in_layers[1].out_tensor
    atom_to_pair = in_layers[2].out_tensor

    AA = tf.matmul(atom_features, self.W_AA) + self.b_AA
    AA = self.activation(AA)
    PA = tf.matmul(pair_features, self.W_PA) + self.b_PA
    PA = self.activation(PA)
    PA = tf.segment_sum(PA, pair_split)

    A = tf.matmul(tf.concat([AA, PA], 1), self.W_A) + self.b_A
    A = self.activation(A)

    if self.update_pair:
      AP_ij = tf.matmul(
          tf.reshape(
              tf.gather(atom_features, atom_to_pair),
              [-1, 2 * self.n_atom_input_feat]), self.W_AP) + self.b_AP
      AP_ij = self.activation(AP_ij)
      AP_ji = tf.matmul(
          tf.reshape(
              tf.gather(atom_features, tf.reverse(atom_to_pair, [1])),
              [-1, 2 * self.n_atom_input_feat]), self.W_AP) + self.b_AP
      AP_ji = self.activation(AP_ji)

      PP = tf.matmul(pair_features, self.W_PP) + self.b_PP
      PP = self.activation(PP)
      P = tf.matmul(tf.concat([AP_ij + AP_ji, PP], 1), self.W_P) + self.b_P
      P = self.activation(P)
    else:
      P = pair_features
    out_tensor = [A, P]
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

class WeaveGather(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.WeaveGather
  """

  def __init__(self,
               batch_size,
               n_input=128,
               gaussian_expand=False,
               init='glorot_uniform',
               activation='tanh',
               epsilon=1e-3,
               momentum=0.99,
               **kwargs):
    """
    Parameters
    ----------
    batch_size: int
      number of molecules in a batch
    n_input: int, optional
      number of features for each input molecule
    gaussian_expand: boolean. optional
      Whether to expand each dimension of atomic features by gaussian histogram
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied

    """
    self.n_input = n_input
    self.batch_size = batch_size
    self.gaussian_expand = gaussian_expand
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.epsilon = epsilon
    self.momentum = momentum
    super(WeaveGather, self).__init__(**kwargs)

  def build(self):
    if self.gaussian_expand:
      self.W = self.init([self.n_input * 11, self.n_input])
      self.b = model_ops.zeros(shape=[
          self.n_input,
      ])
      self.trainable_weights = self.W + self.b
    else:
      self.trainable_weights = None

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """ description and explanation refer to deepchem.nn.WeaveGather
    parent layers: atom_features, atom_split
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    outputs = in_layers[0].out_tensor
    atom_split = in_layers[1].out_tensor

    if self.gaussian_expand:
      outputs = self.gaussian_histogram(outputs)

    output_molecules = tf.segment_sum(outputs, atom_split)

    if self.gaussian_expand:
      output_molecules = tf.matmul(output_molecules, self.W) + self.b
      output_molecules = self.activation(output_molecules)
    out_tensor = output_molecules
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

  def gaussian_histogram(self, x):
    gaussian_memberships = [(-1.645, 0.283), (-1.080, 0.170), (-0.739, 0.134),
                            (-0.468, 0.118), (-0.228, 0.114), (0., 0.114),
                            (0.228, 0.114), (0.468, 0.118), (0.739, 0.134),
                            (1.080, 0.170), (1.645, 0.283)]
    dist = [
        tf.contrib.distributions.Normal(mu=p[0], sigma=p[1])
        for p in gaussian_memberships
    ]
    dist_max = [dist[i].pdf(gaussian_memberships[i][0]) for i in range(11)]
    outputs = [dist[i].pdf(x) / dist_max[i] for i in range(11)]
    outputs = tf.stack(outputs, axis=2)
    outputs = outputs / tf.reduce_sum(outputs, axis=2, keep_dims=True)
    outputs = tf.reshape(outputs, [-1, self.n_input * 11])
    return outputs

class DTNNEmbedding(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DTNNEmbedding
  """

  def __init__(self,
               n_embedding=30,
               periodic_table_length=83,
               init='glorot_uniform',
               **kwargs):
    """
    Parameters
    ----------
    n_embedding: int, optional
      Number of features for each atom
    periodic_table_length: int, optional
      Length of embedding, 83=Bi
    init: str, optional
      Weight initialization for filters.
    """
    self.n_embedding = n_embedding
    self.periodic_table_length = periodic_table_length
    self.init = initializations.get(init)  # Set weight initialization

    super(DTNNEmbedding, self).__init__(**kwargs)

  def build(self):

    self.embedding_list = self.init(
        [self.periodic_table_length, self.n_embedding])
    self.trainable_weights = [self.embedding_list]

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DTNNEmbedding
    parent layers: atom_number
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    atom_number = in_layers[0].out_tensor
    atom_features = tf.nn.embedding_lookup(self.embedding_list, atom_number)
    self.out_tensor = atom_features

class DTNNStep(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DTNNStep
  """

  def __init__(self,
               n_embedding=30,
               n_distance=100,
               n_hidden=60,
               init='glorot_uniform',
               activation='tanh',
               **kwargs):
    """
    Parameters
    ----------
    n_embedding: int, optional
      Number of features for each atom
    n_distance: int, optional
      granularity of distance matrix
    n_hidden: int, optional
      Number of nodes in hidden layer
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    """
    self.n_embedding = n_embedding
    self.n_distance = n_distance
    self.n_hidden = n_hidden
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations

    super(DTNNStep, self).__init__(**kwargs)

  def build(self):
    self.W_cf = self.init([self.n_embedding, self.n_hidden])
    self.W_df = self.init([self.n_distance, self.n_hidden])
    self.W_fc = self.init([self.n_hidden, self.n_embedding])
    self.b_cf = model_ops.zeros(shape=[
        self.n_hidden,
    ])
    self.b_df = model_ops.zeros(shape=[
        self.n_hidden,
    ])

    self.trainable_weights = [
        self.W_cf, self.W_df, self.W_fc, self.b_cf, self.b_df
    ]

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DTNNStep
    parent layers: atom_features, distance, distance_membership_i, distance_membership_j
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    atom_features = in_layers[0].out_tensor
    distance = in_layers[1].out_tensor
    distance_membership_i = in_layers[2].out_tensor
    distance_membership_j = in_layers[3].out_tensor
    distance_hidden = tf.matmul(distance, self.W_df) + self.b_df
    atom_features_hidden = tf.matmul(atom_features, self.W_cf) + self.b_cf
    outputs = tf.multiply(distance_hidden,
                          tf.gather(atom_features_hidden,
                                    distance_membership_j))

    # for atom i in a molecule m, this step multiplies together distance info of atom pair(i,j)
    # and embeddings of atom j(both gone through a hidden layer)
    outputs = tf.matmul(outputs, self.W_fc)
    outputs = self.activation(outputs)

    output_ii = tf.multiply(self.b_df, atom_features_hidden)
    output_ii = tf.matmul(output_ii, self.W_fc)
    output_ii = self.activation(output_ii)

    # for atom i, sum the influence from all other atom j in the molecule
    outputs = tf.segment_sum(outputs,
                             distance_membership_i) - output_ii + atom_features
    out_tensor = outputs
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

class DTNNGather(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DTNNGather
  """

  def __init__(self,
               n_embedding=30,
               n_outputs=100,
               layer_sizes=[100],
               init='glorot_uniform',
               activation='tanh',
               **kwargs):
    """
    Parameters
    ----------
    n_embedding: int, optional
      Number of features for each atom
    n_outputs: int, optional
      Number of features for each molecule(output)
    layer_sizes: list of int, optional(default=[1000])
      Structure of hidden layer(s)
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    """
    self.n_embedding = n_embedding
    self.n_outputs = n_outputs
    self.layer_sizes = layer_sizes
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations

    super(DTNNGather, self).__init__(**kwargs)

  def build(self):
    self.W_list = []
    self.b_list = []
    prev_layer_size = self.n_embedding
    for i, layer_size in enumerate(self.layer_sizes):
      self.W_list.append(self.init([prev_layer_size, layer_size]))
      self.b_list.append(model_ops.zeros(shape=[
          layer_size,
      ]))
      prev_layer_size = layer_size
    self.W_list.append(self.init([prev_layer_size, self.n_outputs]))
    self.b_list.append(model_ops.zeros(shape=[
        self.n_outputs,
    ]))
    prev_layer_size = self.n_outputs

    self.trainable_weights = self.W_list + self.b_list

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DTNNGather
    parent layers: atom_features, atom_membership
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    output = in_layers[0].out_tensor
    atom_membership = in_layers[1].out_tensor
    for i, W in enumerate(self.W_list):
      output = tf.matmul(output, W) + self.b_list[i]
      output = self.activation(output)
    output = tf.segment_sum(output, atom_membership)
    out_tensor = output
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

class DAGLayer(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DAGLayer
  """

  def __init__(self,
               n_graph_feat=30,
               n_atom_feat=75,
               max_atoms=50,
               layer_sizes=[100],
               init='glorot_uniform',
               activation='relu',
               dropout=None,
               batch_size=64,
               **kwargs):
    """
    Parameters
    ----------
    n_graph_feat: int, optional
      Number of features for each node(and the whole grah).
    n_atom_feat: int, optional
      Number of features listed per atom.
    max_atoms: int, optional
      Maximum number of atoms in molecules.
    layer_sizes: list of int, optional(default=[1000])
      Structure of hidden layer(s)
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    dropout: float, optional
      Dropout probability, not supported here
    batch_size: int, optional
      number of molecules in a batch
    """
    super(DAGLayer, self).__init__(**kwargs)

    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.layer_sizes = layer_sizes
    self.dropout = dropout
    self.max_atoms = max_atoms
    self.batch_size = batch_size
    self.n_inputs = n_atom_feat + (self.max_atoms - 1) * n_graph_feat
    # number of inputs each step
    self.n_graph_feat = n_graph_feat
    self.n_outputs = n_graph_feat
    self.n_atom_feat = n_atom_feat

  def build(self):
    """"Construct internal trainable weights.
    """

    self.W_list = []
    self.b_list = []
    prev_layer_size = self.n_inputs
    for layer_size in self.layer_sizes:
      self.W_list.append(self.init([prev_layer_size, layer_size]))
      self.b_list.append(model_ops.zeros(shape=[
          layer_size,
      ]))
      prev_layer_size = layer_size
    self.W_list.append(self.init([prev_layer_size, self.n_outputs]))
    self.b_list.append(model_ops.zeros(shape=[
        self.n_outputs,
    ]))

    self.trainable_weights = self.W_list + self.b_list

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DAGLayer
    parent layers: atom_features, parents, calculation_orders, calculation_masks, n_atoms
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    # Add trainable weights
    self.build()

    atom_features = in_layers[0].out_tensor
    # each atom corresponds to a graph, which is represented by the `max_atoms*max_atoms` int32 matrix of index
    # each gragh include `max_atoms` of steps(corresponding to rows) of calculating graph features
    parents = in_layers[1].out_tensor
    # target atoms for each step: (batch_size*max_atoms) * max_atoms
    calculation_orders = in_layers[2].out_tensor
    calculation_masks = in_layers[3].out_tensor

    n_atoms = in_layers[4].out_tensor
    # initialize graph features for each graph
    graph_features_initial = tf.zeros((self.max_atoms * self.batch_size,
                                       self.max_atoms + 1, self.n_graph_feat))
    # initialize graph features for each graph
    # another row of zeros is generated for padded dummy atoms
    graph_features = tf.Variable(graph_features_initial, trainable=False)

    for count in range(self.max_atoms):
      # `count`-th step
      # extracting atom features of target atoms: (batch_size*max_atoms) * n_atom_features
      mask = calculation_masks[:, count]
      current_round = tf.boolean_mask(calculation_orders[:, count], mask)
      batch_atom_features = tf.gather(atom_features, current_round)

      # generating index for graph features used in the inputs
      index = tf.stack(
          [
              tf.reshape(
                  tf.stack(
                      [tf.boolean_mask(tf.range(n_atoms), mask)] *
                      (self.max_atoms - 1),
                      axis=1), [-1]),
              tf.reshape(tf.boolean_mask(parents[:, count, 1:], mask), [-1])
          ],
          axis=1)
      # extracting graph features for parents of the target atoms, then flatten
      # shape: (batch_size*max_atoms) * [(max_atoms-1)*n_graph_features]
      batch_graph_features = tf.reshape(
          tf.gather_nd(graph_features, index),
          [-1, (self.max_atoms - 1) * self.n_graph_feat])

      # concat into the input tensor: (batch_size*max_atoms) * n_inputs
      batch_inputs = tf.concat(
          axis=1, values=[batch_atom_features, batch_graph_features])
      # DAGgraph_step maps from batch_inputs to a batch of graph_features
      # of shape: (batch_size*max_atoms) * n_graph_features
      # representing the graph features of target atoms in each graph
      batch_outputs = self.DAGgraph_step(batch_inputs, self.W_list, self.b_list)

      # index for targe atoms
      target_index = tf.stack([tf.range(n_atoms), parents[:, count, 0]], axis=1)
      target_index = tf.boolean_mask(target_index, mask)
      # update the graph features for target atoms
      graph_features = tf.scatter_nd_update(graph_features, target_index,
                                            batch_outputs)

    out_tensor = batch_outputs
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

  def DAGgraph_step(self, batch_inputs, W_list, b_list):
    outputs = batch_inputs
    for idw, W in enumerate(W_list):
      outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw])
      outputs = self.activation(outputs)
    return outputs

class DAGGather(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DAGGather
  """

  def __init__(self,
               n_graph_feat=30,
               n_outputs=30,
               max_atoms=50,
               layer_sizes=[100],
               init='glorot_uniform',
               activation='relu',
               dropout=None,
               **kwargs):
    """
    Parameters
    ----------
    n_graph_feat: int, optional
      Number of features for each atom
    n_outputs: int, optional
      Number of features for each molecule.
    max_atoms: int, optional
      Maximum number of atoms in molecules.
    layer_sizes: list of int, optional
      Structure of hidden layer(s)
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    dropout: float, optional
      Dropout probability, not supported
    """
    super(DAGGather, self).__init__(**kwargs)

    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.layer_sizes = layer_sizes
    self.dropout = dropout
    self.max_atoms = max_atoms
    self.n_graph_feat = n_graph_feat
    self.n_outputs = n_outputs

  def build(self):
    """"Construct internal trainable weights.
    """

    self.W_list = []
    self.b_list = []
    prev_layer_size = self.n_graph_feat
    for layer_size in self.layer_sizes:
      self.W_list.append(self.init([prev_layer_size, layer_size]))
      self.b_list.append(model_ops.zeros(shape=[
          layer_size,
      ]))
      prev_layer_size = layer_size
    self.W_list.append(self.init([prev_layer_size, self.n_outputs]))
    self.b_list.append(model_ops.zeros(shape=[
        self.n_outputs,
    ]))

    self.trainable_weights = self.W_list + self.b_list

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DAGGather
    parent layers: atom_features, membership
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    # Add trainable weights
    self.build()

    # Extract atom_features
    atom_features = in_layers[0].out_tensor
    membership = in_layers[1].out_tensor
    # Extract atom_features
    graph_features = tf.segment_sum(atom_features, membership)
    # sum all graph outputs
    outputs = self.DAGgraph_step(graph_features, self.W_list, self.b_list)
    out_tensor = outputs
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

  def DAGgraph_step(self, batch_inputs, W_list, b_list):
    outputs = batch_inputs
    for idw, W in enumerate(W_list):
      outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw])
      outputs = self.activation(outputs)
    return outputs


import random

from motor import Motor
from controller import Controller
from Distance import Distance
from Infrared import Infrared
from Tracking import Tracking
import time
import threading
from LED import LED
import RPi.GPIO as GPIO

GPIO.setwarnings(False)

smp_car = Motor() #初始化电机
smp_car.setup()

pwm1 = smp_car.pwm(smp_car.CONTROL1) #初始化左边车轮控制
pwm2 = smp_car.pwm(smp_car.CONTROL2) #初始化右边车轮控制

smart_car = Controller(smp_car, pwm1, pwm2)

ultr = Distance() #初始化超声波测距
ultr.setup()    #初始化引脚

infrared = Infrared() #初始化红外避障
infrared.setup()  #初始化引脚

tracking = Tracking() #初始化循迹
tracking.setup()  #初始化引脚

exit_ultra = 1
def ultra_control():
    while True:
        while not exit_ultra:
            smart_car.acc_value = 20
            smart_car.accelerator()

            dis = ultr.distance()
            if(dis < 10): #后退0.3s
                smart_car.backword()
            elif(dis < 20):
                infra_left_value, infra_right_value = infrared.infra_detect()
                # print(infra_left_value, " ", infra_right_value)

                # 红外避障控制
                if infra_left_value == 0:  # 左侧有障碍物
                    smart_car.turnRight()

                if infra_right_value == 0:  # 右侧有障碍物
                    smart_car.turnLeft()
                else:                       #随即向左或向右偏移
                    val = random.random()
                    if val > 0.5:
                        smart_car.turnRight()
                    else:
                        smart_car.turnLeft()
            else:
                smart_car.forward()

            time.sleep(0.3)

# def infra_control():
#     smart_car.accelerator(0.5, 0.5)
#     infra_left_value, infra_right_value = infrared.infra_detect()
#     #print(infra_left_value, " ", infra_right_value)
#
#     #红外避障控制
#     if infra_left_value == 0:   #左侧有障碍物
#         smart_car.turnRight()
#         time.sleep(0.5)  # 右转0.5s
#         smart_car.stop()
#
#
#     if infra_right_value == 1: #右侧有障碍物
#         smart_car.turnLeft()
#         time.sleep(0.5)  #左转0.5s
#         smart_car.stop()

exit_tracking = 1 #是否退出小车循迹的while循环的标志，为0则不退出循迹
def trackibg_control():
    #红外传感器，控制小车沿黑线自动行进
    while True:
        time.sleep(0.1)
        while not exit_tracking:
            smart_car.acc_value = 30 #占空比的基值为10
            tracking_left_value, tracking_right_value = tracking.tracking_detect()
            #print(tracking_left_value, " ", tracking_right_value)
            if tracking_left_value == 0 and tracking_right_value == 0: #左右都没有检测到红外反射，停车
                smart_car.stop()

            if tracking_left_value == 1 and tracking_right_value == 0: #右侧没有检测到红外反射，即遇到黑线，右转
                smart_car.accelerator(1, 0)
                smart_car.forward()

            if tracking_left_value == 0 and tracking_right_value == 1: #左侧没有检测到反射信号，即遇到黑线，左转
                smart_car.accelerator(0, 1)
                smart_car.forward()

            if tracking_left_value == 1 and tracking_right_value == 1: #左右都能检测到红外反射，前进
                smart_car.accelerator() #左右车速一致
                smart_car.forward()

if __name__ == "__main__":
    try:
        #创建循迹控制线程
        thread_tracking = threading.Thread(target=trackibg_control)
        thread_tracking.start()

        #创建自动避障线程
        thread_distance = threading.Thread(target=ultra_control)
        thread_distance.start()

        f = open("/dev/input/event0", "rb")
        while True:
            d = f.read(48)
            key = d.hex()[40:42]
            print(key)
            if key == "45": #按下该按键，进入循迹模式
                exit_tracking = 0 #由循迹线程控制小车前进
                exit_ultra = 1
            elif key == "46":
                exit_ultra = 0  #有自动避障功能控制小车前进
                exit_tracking = 1
            else:
                exit_tracking = 1 #结束循迹线程中的while循环
                exit_ultra = 1    #结束自动避障中的while循环

                smart_car.accelerator()
                if key == "08":
                    smart_car.turnLeft()
                elif key == "5a":
                    smart_car.turnRight()
                elif key == "18":
                    smart_car.forward()
                elif key == "52":
                    smart_car.backword()
                elif key == "0c":
                    if(smart_car.acc_value < 80):
                        smart_car.acc_value += 20
                    smart_car.accelerator()
                elif key == "5e":
                    if(smart_car.acc_value > 20):
                        smart_car.acc_value -= 20
                    smart_car.accelerator()
                elif key == "1c":
                    smart_car.accelerator(0, 0)
                elif key == "42":
                    LED()
                elif key == "4a":
                    break
    except KeyboardInterrupt:
        smp_car.destroy(pwm1, pwm2)
    finally:
        smp_car.destroy(pwm1, pwm2)


import sys, os

LeftParenthesis, RightParenthesis = '(', ')'

ALL = {
	'(': True,
	'+': True,
	'-': True,
	'*': True,
	'/': True,
	'P': True,
	'N': True,
	'^': True,
	')': True
}

BINARYOP = {
	'+': True,
	'-': True,
	'*': True,
	'/': True,
	'^': True
}

OPERATOR = {
	'(': True,
	')': True,
	'+': True,
	'-': True,
	'*': True,
	'/': True,
	'^': True
}

PRIORITY = {
	'(': 0,
	'+': 1,
	'-': 1,
	'*': 2,
	'/': 2,
	'P': 3,
	'N': 3,
	'^': 4
}

UNARY = {
	'+': 'P',
	'-': 'N'
}

SYNTAX_ERROR = 'Error: Syntax error! Please check your input.'
DIVISION_BY_ZERO = 'Error: Division by zero.'
OVERFLOW = 'Error: Overflow.'

def Str2Number(str):
	try:
		return float(str)
	except ValueError:
		raise Exception(SYNTAX_ERROR)

def doOp(op, operand):
	if op == 'P' or op == 'N':
		if not operand:
			raise Exception(SYNTAX_ERROR)
		else:
			operand[-1] = operand[-1] if op == 'P' else -operand[-1]
	else:
		if len(operand) < 2:
			raise Exception(SYNTAX_ERROR)
		else:
			try:
				top = operand.pop()
				if op == '+':
					operand[-1] += top
				elif op == '-':
					operand[-1] -= top
				elif op == '*':
					operand[-1] *= top
				elif op == '/':
					operand[-1] /= top
				elif op == '^':
					operand[-1] **= top
				else:
					raise Exception(SYNTAX_ERROR)
			except ZeroDivisionError:
				raise Exception(DIVISION_BY_ZERO)
			except:
				raise Exception(OVERFLOW)

def processOp(op, operator, operand):
	while operator and PRIORITY[op] <= PRIORITY[operator[-1]]:
		if PRIORITY[op] == 3 and PRIORITY[operator[-1]]:
			break
		if PRIORITY[op] == 4 and PRIORITY[operator[-1]] == 4:
			break
		top = operator.pop()
		doOp(top, operand)
	operator.append(op)

def processPa(operator, operand):
	flag = False
	while operator:
		top = operator.pop()
		if top == LeftParenthesis:
			flag = True
			break
		else:
			doOp(top, operand)
	if not flag:
		raise Exception(SYNTAX_ERROR)

def parseExp(exp):
	# exp = exp.replace(' ', '')
	length = len(exp)
	expParse = []
	i = 0
	flag = False
	while i < length:
		if exp[i] == ' ':
			i += 1
			continue
		elif exp[i].isdigit() or exp[i] == '.':
			number = ''
			while (i < length) and (exp[i].isdigit() or exp[i] == '.'):
				number += exp[i]
				i += 1
			i -= 1
			expParse.append(Str2Number(number))
		elif exp[i] in OPERATOR:
			if exp[i] == '+' or exp[i] == '-':
				expParse.append(UNARY[exp[i]] if not flag or expParse[-1] in PRIORITY else exp[i])
			else:
				expParse.append(exp[i])
		else:
			raise Exception(SYNTAX_ERROR)

		i += 1
		flag = True

	return expParse

def evalExp(exp):
	expParse = parseExp(exp)
	operator, operand = [], []
	pre = None
	for s in expParse:
		if ((s in BINARYOP or s == ')') and pre in BINARYOP) or (s in BINARYOP and (pre in BINARYOP or pre == '(')):
			raise Exception(SYNTAX_ERROR)
		if pre is None and (s in BINARYOP or s == ')'):
			raise Exception(SYNTAX_ERROR)
		if pre is not None and (pre not in ALL and s not in ALL):
			raise Exception(SYNTAX_ERROR)
		if s == LeftParenthesis:
			operator.append(s)
		elif s == RightParenthesis:
			processPa(operator, operand)
		elif s in PRIORITY:
			processOp(s, operator, operand)
		else:
			operand.append(s)
		pre = s

	while operator:
		doOp(operator.pop(), operand)

	if len(operand) != 1:
		raise Exception(SYNTAX_ERROR)

	return operand[-1]

# exp = '((2^-2^-2^2^(+-+-2*42/4567^2)-791314/798746)+134)'
# print(parseExp(exp))
# print("my: ")
# evalExp(exp)
# print("correct: ")
# check = exp.replace('^', '**')
# command = 'python3 -qc \"print(' + check + ')\"'
# os.system(command)


from PyQt5 import QtWidgets
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPixmap

from gui.edge_based_dices_detection_controller import EdgeBasedDicesDetectionController
import numpy as np

class EdgeBasedDicesDetectionScreen(QtWidgets.QWidget):
    def __init__(self, title: str):
        super().__init__()
        self.setWindowTitle(title)
        self._init_gui()
        self._create_controller()

    def _init_gui(self):
        available_geometry = QtWidgets.QDesktopWidget().availableGeometry()
        screen_center = available_geometry.center()

        self.width = available_geometry.width() // 2
        self.height = available_geometry.height() // 3

        x = screen_center.x() - self.width // 2
        y = screen_center.y() - self.height // 2

        self.setGeometry(x, y, self.width, self.height)
        self.image = QtWidgets.QLabel(self)
        self.image.resize(self.width // 2, self.height)
        # temporary, remove later
        self.test_slider1 = QtWidgets.QSlider(Qt.Horizontal, self)
        self.test_slider1.setMaximumWidth(self.width // 2)
        self.test_slider2 = QtWidgets.QSlider(Qt.Horizontal, self)
        self.test_slider2.setMaximumWidth(self.width // 2)
        self.test_slider3 = QtWidgets.QSlider(Qt.Horizontal, self)
        self.test_slider3.setMaximumWidth(self.width // 2)

        main_h_box = QtWidgets.QHBoxLayout()

        main_h_box.addWidget(self.image)

        instrument_panel_v_box = QtWidgets.QVBoxLayout()
        instrument_panel_v_box.addWidget(self.test_slider1)
        instrument_panel_v_box.addWidget(self.test_slider2)
        instrument_panel_v_box.addWidget(self.test_slider3)

        main_h_box.addLayout(instrument_panel_v_box)

        self.setLayout(main_h_box)

    def _create_controller(self):
        self._controller = EdgeBasedDicesDetectionController()
        self._controller.images.connect(self._on_image_received)

    def _on_image_received(self, image: np.ndarray):
        scaled = image.scaled(self.width // 2, self.height, Qt.KeepAspectRatio)
        pix_map = QPixmap.fromImage(scaled)
        self.image.setPixmap(pix_map)

    def closeEvent(self, event):
        self._controller.clear_resources()
        event.accept()


from bs4 import BeautifulSoup
import requests
import csv
import re

headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"}

producct_details = []
def get_page(url):
    html_text = requests.get(url,headers=headers).text
    soup = BeautifulSoup(html_text, "lxml")
    return soup

def get_product_details(url):
    manufacturer = ""
    soup = get_page(url)
    description = soup.find("span", id="productTitle").text.strip()
    try:
        product_description = soup.find("div", id="productDescription")
        productDescription = product_description.find("p").text.strip()
    except:
        productDescription = ""

    try:
        deails = soup.find("div", id="detailBullets_feature_div")
        details = deails.find_all("li")
        for detail in details:
            if re.search("^Manufacturer",detail.text.strip()):
                manufacturer = detail.find("span", class_="").text.strip()
                break
    except:
        manufacturer = ""
    try:
        deails = soup.find("div", id="detailBullets_feature_div")
        details = deails.find_all("li")
        for detail in details:
            if re.search("^ASIN",detail.text.strip()):
                asin = detail.find("span", class_="").text.strip()
                break
    except:
        asin = ""

    return (description,asin,productDescription,manufacturer)

def get_record(item):
    try:
        item_name = item.find("span", class_="a-size-medium a-color-base a-text-normal").text
    except AttributeError:
        item_name = ""

    try:
        item_price = item.find("span", class_="a-price-whole").text
    except AttributeError:
        item_price = ""

    try:
        rating = item.find("span", class_="a-icon-alt").text
    except AttributeError:
        rating = ""

    try:
        review_count = item.find("span", class_="a-size-base s-underline-text").text
    except AttributeError:
        review_count = ""

    url = item.find("a", class_="a-link-normal s-underline-text s-underline-link-text s-link-style a-text-normal").get("href")
    url = "https://www.amazon.in" + url

    result = (url, item_name, item_price, rating, review_count)
    return result

if __name__ == "__main__":

    url = "https://www.amazon.in/s?k=bags&crid=2M096C61O4MLT&qid=1653308124&sprefix=ba%2Caps%2C283&ref=sr_pg_1"
    records = []
    pno = 1

    for _ in range(20):
        print("Scraping page: " + str(pno))
        soup = get_page(url)
        items = soup.find_all("div", {"data-component-type": "s-search-result"})

        for item in items:
            record = get_record(item)
            if record:
                records.append(record)
        if pno != 20:
            next_link = soup.find("a", class_ = "s-pagination-item s-pagination-next s-pagination-button s-pagination-separator").get("href")
            url = "https://www.amazon.in" + next_link
        pno += 1

    print("Scraping completed. Total records: " + str(len(records)))

    with open("amazonsearchresult.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(["URL", "Name", "Price", "Rating", "Review Count"])
        writer.writerows(records)
    print("CSV file for search created successfully.")

    with open("amazonproductdetails.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(["Description", "ASIN", "Product Description", "manufacturer"])
        i=0
        for record in records:
            product_details = get_product_details(record[0])
            writer.writerow(product_details)
            print("Product details for " + str(i) + " scraped successfully.")
            i+=1

    print("CSV file created successfully.")


import sys
import re
import urllib2
import csv

"""
Momente si spite
"""
if __name__ == "__main__":
    url = 'http://www.cotidianul.ro/rss'
    pattern = r'<a.+?href="(?P<rsslink>.+?\.xml)".*?>(?P<rsstitle>.+?)'
    pattern = re.compile(pattern)

    conn = urllib2.urlopen(url)
    data = conn.read()
    found = pattern.findall(data)
    writer = csv.writer(open('cotidianul.csv', 'w'), delimiter='\t', quotechar='"')
    for link, title in found:
        if not link.startswith('http'):
            link = 'http://www.cotidianul.ro/rss/' + link
        writer.writerow([link, title])


from django.core.management.base import BaseCommand, CommandError
from tree_sitter import Language, Parser, Tree

from plagiarism.tokens import Token, parse_tree
from plagiarism.sources import parse_source, match_sequences, tokenize_source

class Command(BaseCommand):
    def handle(self, *args, **options):
        first_source: str = """
        Test = 9
        def foo():
            def main():
                potato = potato + 1
                if bar:
                    baz()
            if bar:
                baz()
        Test = 10
        """
        second_source: str = """
        SomethingElse = 9
        def main():
            if bar:
                baz()
            def foo():
                potato += 1
                if bar:
                    baz()
        SomethingElse = 10
        """

        # Generate source tree
        first_tree: Tree = parse_source(source=first_source, ext=".py")
        second_tree: Tree = parse_source(source=second_source, ext=".py")

        # Parse tree to generate list of tokens
        first_parse: list[Token] = parse_tree(first_tree.walk(), is_named=None, child_only=False)
        second_parse: list[Token] = parse_tree(second_tree.walk(), is_named=None, child_only=False)

        # Displaying the two lists of tokens side by side
        first = first_parse if len(first_parse) > len(second_parse) else second_parse
        first_taken_size = min(len(first_parse), len(second_parse))
        print("_________________________________\n")
        for fp, sp in zip(first_parse, second_parse):
            print(f"{str(fp):<20} | {str(sp)}")
        for leftover in first[first_taken_size:]:
            if first == first_parse:
                print(f"{str(leftover):<20} |")
            else:
                print(f"{' ':<20} | {str(leftover)}")

        print("_________________________________\n")

        # Detect plagiarism between two list of tokens and mark similarity in source code
        marked_first_source, marked_second_source, plag_ratio = match_sequences(
            tokens1=first_parse, tokens2=second_parse, source1=first_source, source2=second_source
        )
        print(
            # Display highlighted plagiarism
            tokenize_source(
                source=first_source, marked_source=marked_first_source, start_token="\033[94m", end_token="\033[0m"
            )
        )
        print("\n----------")
        print(
            # Display highlighted plagiarism
            tokenize_source(
                source=second_source, marked_source=marked_second_source, start_token="\033[94m", end_token="\033[0m"
            )
        )
        print(f"\n\nPlagiarism Percentage: {plag_ratio*100:.5}%")