#!/usr/bin/env python
# coding: utf-8

# In[3]:

import csv
import locale
a = []
b = []
c = []
with open('T86_ALLBUT0999_20211208.csv', encoding='big5' , errors = 'ignore') as a1:
    rows = csv.reader(a1, delimiter=',')
    a += rows
    for j in range(3,121):
        errors = 'ignore'
        if a[j][4].replace(',','') =='----' or locale.atof(a[j][10].replace(',','')) =='----' :
            print('wrong')
        elif  float(a[j][14].replace(',',"")) >= 50000:
            print('第一日',a[j][0],a[j][1],a[j][14])
with open('T86_ALLBUT0999_20211209.csv', encoding='big5' , errors = 'ignore') as a2:
    rows1 = csv.reader(a2, delimiter=',')
    b += rows1
    for j in range(3,121):
        errors = 'ignore'
        if b[j][4].replace(',','') =='----' or locale.atof(b[j][10].replace(',','')) =='----' :
            print('wrong')
        elif  locale.atof(b[j][10].replace(',',"")) >= 50000:
            print('第二日',b[j][0],b[j][1],b[j][10])
with open('T86_ALLBUT0999_20211210.csv', encoding='big5' , errors = 'ignore') as a3:
    rows2 = csv.reader(a3, delimiter=',')
    c += rows2
    for j in range(3,121):
        errors = 'ignore'
        if c[j][4].replace(',','') =='----' or locale.atof(c[j][10].replace(',','')) =='----' :
            print('wrong')
        elif  locale.atof(c[j][10].replace(',',"")) >= 50000:
            print('第三日',c[j][0],c[j][1],c[j][10])

# In[ ]:

# In[ ]:

# In[ ]:


# -*- coding: utf-8 -*-
#harvest theses from TCD, Dublin
#FS: 2020-09-02
#FS: 2023-03-09

import getopt
import sys
import os
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
import re
import ejlmod3
import time

years = [ejlmod3.year(backwards=1), ejlmod3.year()]
rpp = 50
pages = 20
numberofrecords = rpp*pages
skipalreadyharvested = True

publisher = 'TCD, Dublin'
jnlfilename = 'THESES-TrinityCollegeDublin-%s' % (ejlmod3.stampoftoday())

boringschools = ['Biochemistry & Immunology', 'Business', 'Chemistry',
                 'Creative Arts', 'Dental Sciences', 'Religion',
                 'Ecumenics', 'Education', 'Engineering', 'English',
                 'Histories & Humanities', 'Lang, Lit', 'Law',
                 'Linguistic Speech & Comm Sci', 'Medicine', 'Natural Sciences',
                 'Nursing & Midwifery', 'Pharmacy & Pharma', 'Psychology',
                 'Social Sciences & Philosophy', 'Social Work & Social Policy']

if skipalreadyharvested:
    alreadyharvested = ejlmod3.getalreadyharvested(jnlfilename)

hdls = []

prerecs = []
for year in years:
    for section in ['76240', '221', '171']:
        realpages = pages
        for page in range(pages):
            if page*rpp < numberofrecords:
                tocurl = 'http://www.tara.tcd.ie/handle/2262/' + section + '/discover?rpp=' + str(rpp) + '&page=' + str(page+1) + '&group_by=none&etal=0&filtertype_0=dateIssued&filtertype_1=type&filter_0=[' + str(year) + '+TO+' + str(year) + ']&filter_relational_operator_1=equals&filter_1=Thesis&filter_relational_operator_0=equals'
                ejlmod3.printprogress("=", [[year], [section], [page+1, realpages]])
                try:
                    tocpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(tocurl), features="lxml")
                    time.sleep(3)
                except:
                    print("retry %s in 180 seconds" % (tocurl))
                    time.sleep(180)
                    tocpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(tocurl), features="lxml")
                if page == 0:
                    for p in tocpage.body.find_all('p', attrs = {'class' : 'pagination-info'}):
                        if re.search('\d of \d+', p.text):
                            numberofrecords = int(re.sub('.*of (\d+).*', r'\1', p.text.strip()))
                            print('  %i theses in query' % (numberofrecords))
                            realpages = (numberofrecords-1) // rpp + 1
                for rec in ejlmod3.getdspacerecs(tocpage, 'http://www.tara.tcd.ie'):
                    if rec['hdl'] in hdls or rec['hdl'] in ['2262/82940']:
                        print('  HDL:%s already from other collection' % (rec['hdl']))
                    elif skipalreadyharvested and rec['hdl'] in alreadyharvested:
                        print('  HDL:%s already in backup' % (rec['hdl']))
                    else:
                        prerecs.append(rec)
                        hdls.append(rec['hdl'])
        print('  %4i records so far' % (len(prerecs)))

i = 0
recs = []
for rec in prerecs:
    keepit = True
    i += 1
    ejlmod3.printprogress("-", [[year], [i, len(prerecs)], [rec['link']], [len(recs)]])
    try:
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['link'] + '?show=full'), features="lxml")
        time.sleep(3)
    except:
        print("retry %s in 180 seconds" % (rec['link'] + '?show=full'))
        time.sleep(180)
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['link']), features="lxml")
    ejlmod3.metatagcheck(rec, artpage, ['citation_authors', 'DCTERMS.issued', 'DCTERMS.abstract',
                                        'DC.subject'])
    for meta in artpage.head.find_all('meta', attrs = {'name' : 'DC.publisher'}):
        if re.search('School of', meta['content']):
            rec['school'] = re.sub('.*School of (.*?)\..*', r'\1', meta['content'])
        else:
            rec['note'].append(meta['content'])
    if 'school' in list(rec.keys()):
        if rec['school'] == 'Mathematics':
            rec['autaff'][-1].append('Trinity Coll., Dublin')
            rec['fc'] = 'm'
        elif rec['school'] in boringschools:
            print('   skip School of %s' % (rec['school']))
            keepit = False
        else:
            rec['note'].append(rec['school'])
    if len(rec['autaff'][-1]) == 1:
        rec['autaff'][-1].append(publisher)
    #license
    ejlmod3.globallicensesearch(rec, artpage)
    #fulltext
    for div in artpage.body.find_all('div', attrs = {'class' : 'simple-item-view-download'}):
        for a in div.find_all('a'):
            if 'license' in list(rec.keys()):
                rec['FFT'] = a['href']
            else:
                rec['hidden'] = a['href']
    #supervisor
    for tr in artpage.body.find_all('tr', attrs = {'class' : 'ds-table-row'}):
        tdt = ''
        for td in tr.find_all('td'):
            if td.has_attr('label-cell'):
                tdt = td.text.strip()
            else:
                if tdt == 'dc.contributor.advisor':
                    rec['supervisor'].append(td.text.strip())

    if keepit:
        recs.append(rec)
        ejlmod3.printrecsummary(rec)
    else:
        ejlmod3.adduninterestingDOI(rec['hdl'])

ejlmod3.writenewXML(recs, publisher, jnlfilename)


from database import Database
from module.post import Posts
import datetime
import uuid

class Users(object):

    def __init__(self,author="none",user_id=uuid.uuid4().hex,discription="none"):
        self.author = author
        self.user_id = user_id
        self.discription = discription
        self.save_to_db()

    def new_post(self,title="none",date=datetime.datetime.utcnow(),content="none"):
        date = datetime.datetime.strptime(date,"%d%m%Y")
        post = Posts(title=title,date=date,user_id=self.user_id,content=content)

    def save_to_db(self):
        retJson = self.json()
        Database.insert("users",retJson)

    def json(self):
        return {
            "author": self.author,
            "user_id": self.user_id,
            "discription": self.discription
        }


# py -m pip install dnspython

import dns.resolver, dns.reversename

strDomains = """
ya.ru
yawdadad1.ru
2020.kbsh.rzd.ru
"""

strIps = """
8.8.8.8
83.234.113.47
"""

domains = strDomains.split()
ips = strIps.split()

for ip in ips:
    addr = dns.reversename.from_address(ip)
    try:
        A = dns.resolver.resolve(addr, 'PTR')
    except dns.resolver.NXDOMAIN:
        print(ip," XXX")
    except Exception:
        print(ip," неизвестная ошибка")
    else:
        for i in A.response.answer:
            for j in i.items:
                print(ip," ",j)

for domain in domains:
    try:
        A = dns.resolver.resolve(domain, 'A')
    except dns.resolver.NXDOMAIN:
        print(domain," 0.0.0.0")
    except Exception:
        print(domain," неизвестная ошибка")
    else:
        for i in A.response.answer:
            for j in i.items:
#                print(domain," ",j.address)
                print(domain," ",j)


"""Module for dealing with WTPH input configs."""
import os
import shutil

import constants as c
from run_numbers import RUN_NUMBERS

def check_formatted():
    """Check that input configs have correct naming scheme, break if not."""
    input_configs = os.listdir(c.WTPH_INPUT_CONF_DIR)
    for year in c.YEARS:
        for period in RUN_NUMBERS[year]:
            for data_mc in ["data", "mc"]:
                assert c.WTPH_INPUT_CONF_FMT[data_mc].format(
                    year=year, period=period) in input_configs

def replace_16_20():
    """
    Replace mc16 with mc20.
    Really we should fix CreateInputConfigs,
    but changes get merged so damn slow.
    """
    for year in c.YEARS:
        for period in RUN_NUMBERS[year]:
            for data_mc in ["data", "mc"]:
                config = c.WTPH_INPUT_CONF_FMT[data_mc].format(
                    year=year, period=period)
                config_path = os.path.join(c.WTPH_INPUT_CONF_DIR, config)
                with open(config_path, "r") as config_file:
                    text = config_file.read()
                text = text.replace("mc16", "mc20")
                with open(config_path, "w") as config_file:
                    config_file.write(text)

def merge_two(config_path_1, config_path_2, output_path):
    """
    Merge two input config files, e.g. two data files of the same period.

    Currently this doesn't preserve info from comments, maybe implement later.
    """

    # get lines from files
    with open(config_path_1, "r") as conf1:
        conf1_lines = conf1.readlines()
    with open(config_path_2, "r") as conf2:
        conf2_lines = conf2.readlines()
    lines = conf1_lines + conf2_lines

    # get all lines of each type, as sets so they're unique
    inputfiles = list({l for l in lines if l.split()[0] == "Input"})
    grls = list({l for l in lines if l.split()[0] == "GRL"})
    prw_data = list({l for l in lines if l.split()[0] == "PRWDataFile"})
    prw_mc = list({l for l in lines if l.split()[0] == "PRWMCFile"})

    new_lines = [
        "############################################",
        "# This is the result of merging two files:",
        "# " + config_path_1,
        "# " + config_path_2,
        "############################################",
        "#InputFiles:"] + inputfiles +\
        ["#Good run Lists to be applied:"] + grls +\
        ["#Pile-up reweighting lumicalc files:"] + prw_data +\
        ["#Pile-up reweighting config files:"] + prw_mc

    with open(output_path, "w") as outfile:
        outfile.writelines(new_lines)
    print("wrote", output_path)

def merge(config_filenames, output_path):
    """Merge a list of input configs."""
    if len(config_filenames) < 2:
        raise ValueError("Why are you trying to merge this list?")
    elif len(config_filenames) == 2:
        merge_two(
            os.path.join(c.WTPH_INPUT_CONF_DIR, config_filenames[0]),
            os.path.join(c.WTPH_INPUT_CONF_DIR, config_filenames[1]),
            output_path)
    else:
        merge(
            [merge(*config_filenames[0:2])] + config_filenames[2:],
            output_path)

def format_input_configs():
    """Ensure that a config exists for data or MC for all years and periods."""
    input_configs = os.listdir(c.WTPH_INPUT_CONF_DIR)
    for year in c.YEARS:
        for period in RUN_NUMBERS[year]:
            # first deal with data files
            data_filename = c.WTPH_INPUT_CONF_FMT["data"].format(
                year=year, period=period)
            data_filepath = os.path.join(c.WTPH_INPUT_CONF_DIR, data_filename)
            # we will never have files with this exact format auto-generated
            if os.path.exists(data_filepath):
                os.remove(data_filepath)
            if data_filename not in input_configs:
                # try looking for a single file with year_period_
                data_matches = [
                    f for f in input_configs
                    if f.startswith("data_{year}_{period}".format(
                        year=year, period=period))]
                if data_matches == []:
                    # try looking for AllYear config
                    data_matches = [
                        f for f in input_configs
                        if f.startswith("data_{year}_AllYear".format(
                            year=year))]
                    assert len(data_matches) == 1
                    filepath = os.path.join(
                        c.WTPH_INPUT_CONF_DIR, data_matches[0])
                    shutil.copyfile(
                        filepath,
                        data_filepath)
                elif len(data_matches) == 1:
                    # if we found a single year_period file:
                    filepath = os.path.join(
                        c.WTPH_INPUT_CONF_DIR, data_matches[0])
                    shutil.copyfile(
                        filepath,
                        data_filepath)
                else:
                    # merge multiple year_period files
                    data_matches = [
                        f for f in input_configs
                        if f.startswith("data_{year}_{period}".format(
                            year=year, period=period))]
                    # merge all the configs for that period into one
                    merge(data_matches, data_filepath)

            # same vibe with mc things
            mc_filename = c.WTPH_INPUT_CONF_FMT["mc"].format(
                year=year, period=period)
            mc_filepath = os.path.join(c.WTPH_INPUT_CONF_DIR, mc_filename)
            # we will never have files with this exact format auto-generated
            if os.path.exists(mc_filepath):
                os.remove(mc_filepath)
            if mc_filename not in input_configs:
                # try looking for a single file with year_period_
                mc_matches = [
                    f for f in input_configs
                    if f.startswith("{year}_{period}_Zmumu".format(
                        year=year, period=period))]
                if mc_matches == []:
                    # try looking a year-long config
                    mc_matches = [
                        f for f in input_configs
                        if f.startswith("{year}_Zmumu".format(
                            year=year))]
                    assert len(mc_matches) == 1
                    filepath = os.path.join(
                        c.WTPH_INPUT_CONF_DIR, mc_matches[0])
                    shutil.copyfile(
                        filepath,
                        mc_filepath)
                elif len(mc_matches) == 1:
                    # if we found a single year_period file:
                    filepath = os.path.join(
                        c.WTPH_INPUT_CONF_DIR, mc_matches[0])
                    shutil.copyfile(
                        filepath,
                        mc_filepath)
                else:
                    # merge multiple year_period files
                    mc_matches = [
                        f for f in input_configs
                        if f.startswith("{year}_{period}_Zmumu".format(
                            year=year, period=period))]
                    # merge all the configs for that period into one
                    merge(mc_matches, mc_filepath)

    check_formatted()
    replace_16_20()


import os
import sys
from datetime import datetime
from datetime import timedelta
from decimal import Decimal

import matplotlib.pyplot as plt
import numpy as np

#python create_fake_data.py 500
#--creates 500 fake data files with internally specified ranges/distributions

def random():
    return np.random.random()

def random_array(size=1):
    return np.random.random(size)

class GenerateIVdata():
    def __init__(self):
        self.LC_at_vfd_plus50v = None
        self.LC_at_500v = None
        self.vbd = random()*500 + 400
        self.voltages = []
        self.currents = []
        self.points = self.gen_points()
        self.LC_at_700 = self.currents[-1]
        self.stability = self.gen_stability_data()
    def gen_stability_data(self):
        stability = []
        noise = 1e-8
        for i in range(3):
            c = self.LC_at_700
            if c != 'O.L.':
                c += random()*2*noise-noise
            stability.append(c)
        return stability
    def gen_points(self):
        vbd = self.vbd
        voltages = np.arange(20,720,20)
        start_current = 1.0e-7

        #curve 1--20 to vbd
        slope1 = random()*1e-10 + 1e-10
        currents = start_current + (voltages-20)*slope1
        noise = random_array(len(currents))*2e-9 - 1e-9
        currents = currents + noise
        if vbd >= 700:
            #get LC_at_500v then return points
            for i in range(len(voltages)):
                v,c = voltages[i],currents[i]
                if v == 500:
                    if c == "O.L.":
                        self.LC_at_500v = c
                    else: #c is float
                        self.LC_at_500v = "{:.4f}".format(c*1e6)
                    break
            self.voltages = list(voltages)
            self.currents = list(currents)
            return list(zip(voltages,currents))

        #curve 2--vbd to 700
        LC_at_vbd = start_current + (vbd-20)*slope1
        slope2 = 1e-7
        overloads = []
        for i in range(len(voltages)):
            v,c = voltages[i],currents[i]

            if v == 500:
                if c == 'O.L.':
                    self.LC_at_500v = c
                else:
                    self.LC_at_500v = "{:.4f}".format(c * 1e6)
            if v >= vbd:
                currents[i] = LC_at_vbd + slope2*(v-vbd)
                if currents[i] > 1e-5:
                    overloads.append(i)

        self.voltages = list(voltages)
        self.currents = list(currents)
        for overload_ind in overloads:
            print(self.currents[overload_ind],"too high! Corresponding to voltage {}".format(self.voltages[overload_ind]))
            self.currents[overload_ind] = "O.L."

        return list(zip(self.voltages,self.currents))

    def plotIV(self,fileind):
        print("fileind =",fileind)
        currents = list(self.currents)
        for i in range(len(currents)):
            c = currents[i]
            if c == 'O.L.':
                currents[i] = 10
            else:
                currents[i] *= 1e6

        plt.plot(self.voltages,currents,'-',label=str(fileind))
    def get_LC_at_vfd_plus_50v(self,deplation_volts):
        for i in range(len(self.voltages)):
            if self.voltages[i] >= deplation_volts + 50:
                c = self.currents[i]
                if c == 'O.L.':
                    return c
                else:
                    return "{:.4f}".format(c * 1e6)
        print("ERROR FINDING LC_at_vfd_plus_50v")

def rand_date(startstr,endstr):
    start_date = datetime.strptime(startstr,"%d/%m/%Y")
    end_date = datetime.strptime(endstr,"%d/%m/%Y")
    days = (end_date - start_date).days
    rand_days = int(random() * days)
    random_date = start_date + timedelta(days=rand_days)
    return datetime.strftime(random_date,"%d/%m/%Y")

def rand_nm(type_ind):
    #type_ind=0 -> barrel, 1 -> EC
    value = ""
    if type_ind == 0:
        n = int(round(random()*3+1))
    else:
        n = int(round(random()*3))

    choices = ["none"]*8+["other"]*2
    npairs = 0
    choice = choices[int(random()*len(choices))]
    if choice == "other":
        npairs = int(random()*20+1)
        mstart = int(random()*(1539-npairs))
        if npairs > 1:
            for i in range(npairs-1):
                m = mstart + i
                value += f"{n}-{m}, "
        #if not none, add final pair w/o trailing comma
        m = mstart + npairs
        value += f"{n}-{m}"
    else:
        value = "none"

    return value,npairs

def random_ingotnum():
    first6str = ""
    for i in range(6):
        first6str += str(int(random()*10))
    second3str = ""
    for i in range(3):
        second3str += str(int(random()*10))
    return first6str + "-" + second3str

def random_ingotnums(n):
    ingotnums = []
    for i in range(n):
        ingotnums.append(random_ingotnum())
    return ingotnums

def write_random_datafiles(nfiles,dirpath):
    os.makedirs(dirpath,exist_ok=True)
    id_prefixes = ["20USBSX","20USESX"]
    ingotnums = random_ingotnums(nfiles // 25)
    ingot_ind = 0
    for file_ind in range(100,100+nfiles):
        print("File_ind = ",file_ind)
        ivdata = GenerateIVdata()
        ivdata.plotIV(file_ind)
        msg = "#General information ITEM section\n"
        msg += "%ITEM\n"

        type_ind = int(round(random())) #0 = barrel, 1 = EC
        idnum = id_prefixes[type_ind] + "%07d"%(file_ind)
        msg += f"Identification Number\t{idnum}\n"
        serialnum = "VPX9000" + str(type_ind+1) + "-" + str(file_ind)
        msg += f"Serial Number\t{serialnum}\n"

        sensor_type = "ATLASDUMMY18"
        msg += f"Sensor Type\t{sensor_type}\n\n"
        msg += "#Test information Test section\n"
        msg += "%TEST\n"
        test_date = rand_date("06/01/2020","09/01/2020")
        msg += f"Test Date (DD/MM/YYYY)\t{test_date}\n"
        problem = (["NO"]*9+["YES"])[int(random()*10)]
        msg += f"PROBLEM\t{problem}\n"
        if problem == "YES": passed = "NO"
        else: passed = "YES"
        msg += f"PASSED\t{passed}\n\n"

        #DATA####################################
        msg += "#Test data Data section\n"
        msg += "%DATA\n"
        iv_temp = int(round(random()*10) + 20)
        msg += f"IV Temperature(C)\t{iv_temp}\n"
        iv_humid = int(round(random()*40) + 20)
        msg += f"IV Humidity(%)\t{iv_humid}\n"
        deplation_volts = int(round(random()*400 + 100))
        msg += f"Deplation Volts (V)\t{deplation_volts}\n"
        LC_at_vfd_plus_50v = ivdata.get_LC_at_vfd_plus_50v(deplation_volts)
        msg += "Leakage Current at Vfd + 50V (microA)\t{}\n".format(LC_at_vfd_plus_50v)
        LC_at_500v = ivdata.LC_at_500v
        msg += "Leakage current at 500 V (microA)\t{}\n".format(LC_at_500v)
        ingotnum = ingotnums[ingot_ind]
        msg += f"Substrate Lot No.\t{ingotnum}\n"
        substrate_type = ["FZ1","(FZ1)","FZ-1"][int(random()*3)]
        msg += f"Substrate Type\t{substrate_type}\n"
        substrate_orient = "100"
        msg += f"Substrate Orient\t{substrate_orient}\n"
        substrate_RUpper = "{:.1f}".format(random()*8.5+3.5)
        msg += f"Substrate R Upper (kOhm.cm)\t{substrate_RUpper}\n"
        substrate_RLower = "{:.1f}".format(random()*5 + 3)
        msg += f"Substrate R Lower (kOhm.cm)\t{substrate_RLower}\n"
        active_thickness = "290"
        msg += f"Active thickness (nominal value)\t{active_thickness}\n"
        polysilicon_bias_upper = "{:.2f}".format(random()+1)
        msg += "Polysilicon Bias Resistance Upper (MOhm)\t{}\n".format(polysilicon_bias_upper)
        polysilicon_bias_lower = "{:.2f}".format(random()+1)
        msg += "Polysilicon Bias Resistance Lower (MOhm)\t{}\n".format(polysilicon_bias_lower)
        onsetV_of_microdischarge = int(round(ivdata.vbd))
        msg += "Onset voltage of Microdischarge (V)\t{}\n\n\n\n\n".format(onsetV_of_microdischarge)

        #DEFECTS##############################
        msg += "#Defects section\n"
        msg += "%DEFECT\n"
        msg += "#DEFECT NAME\n"
        oxide_pinholes,n1 = rand_nm(type_ind)
        msg += f"Oxide pinholes\t{oxide_pinholes}\n"
        metal_shorts,n2 = rand_nm(type_ind)
        msg += f"Metal Shorts\t{metal_shorts}\n"
        metal_opens,n3 = rand_nm(type_ind)
        msg += f"Metal Opens\t{metal_opens}\n"
        implant_shorts,n4 = rand_nm(type_ind)
        msg += f"Implant Shorts\t{implant_shorts}\n"
        if type_ind == 1: #EC
            bias_resistor_disconnection,n5 = rand_nm(type_ind)
            msg += "Bias resistor disconnection\t{}\n".format(bias_resistor_disconnection)
            total_defects = n1+n2+n3+n4+n5
            percent_ng_strips = total_defects / (4*1536)
        if type_ind == 0: #barrel
            implant_opens,n5 = rand_nm(type_ind)
            msg += f"Implant Opens\t{implant_opens}\n"
            microdischarge_strips,n6 = rand_nm(type_ind)
            if microdischarge_strips == "none":
                microdischarge_strips = "-"
            msg += "Microdischarge strips\t{}\n".format(microdischarge_strips)
            total_defects = n1+n2+n3+n4+n5+n6
            percent_ng_strips = total_defects / (4*1282)

        # VF, 2021-08-14: this is actually a fraction, not percentage -> will remove the "%" sign
        #msg += "Percentage of NG strips\t{:.10f}%\n\n".format(percent_ng_strips)
        msg += "Percentage of NG strips\t{:.10f}\n\n".format(percent_ng_strips)

        #RAWDATA########################################
        msg += "#IV raw data Raw data section\n"
        msg += "%RAWDATA\n"
        temp = iv_temp
        msg += f"IV Temperature(C)\t{temp}\n"
        humid = iv_humid
        msg += f"Humidity (%)\t{humid}\n"
        voltage_step = "20"
        msg += f"Voltage step (V)\t{voltage_step}\n"
        delay_time = "1"
        msg += f"Delay time (second)\t{delay_time}\n"
        msg += "IV Characteristics (A)\n"
        msg += "#IV\n"
        for v,c in ivdata.points:
            #'20\t1.23E-07\n'
            if c == 'O.L.':
                msg += f'{v}\t{c}\n'
            else:
                msg += f'{v}\t%.2E\n' % Decimal(c)
        msg += "\n"

        #stability at 700V####################
        msg += "Id stability test @ 700V\n"
        for i in range(3):
            sec = [10,20,30][i]
            c = ivdata.stability[i]
            if c == 'O.L.':
                msg += f'{sec} sec\t{c}\n'
            else:
                msg += f'{sec} sec\t%.2E\n' % Decimal(c)
        msg += "#End of manufacturer data file"
        with open(os.path.join(dirpath,idnum+'.txt'),'w') as f:
            f.write(msg)

        #increase ingot ind if written 25 files#####
        if (file_ind+1) % 25 == 0:
            ingot_ind += 1
    #pre-breakdown plot
    plt.ylim(0,1)
    plt.title("All IV curves (zoomed in)")
    plt.ylabel("Current ($\mu$A)")
    plt.xlabel("Voltage (V)")
    plt.savefig("dummyfiles/all_IVs_zoom.png")
    #post-breakdown plot
    plt.ylim(0,10)
    plt.hlines(10,0,700,linestyle='--')
    plt.title("All IV curves (full scale)")
    plt.ylabel("Current ($\mu$A)")
    plt.xlabel("Voltage (V)")
    plt.savefig("dummyfiles/all_IVs_full.png")

def main():
    #go thru all non-null keys and assign appropriate random values to them
    try:
        nfiles = int(sys.argv[1])
        write_random_datafiles(nfiles,"dummyfiles/")
    except Exception as e:
        raise(e)
        print("Usage: python create_fake_data.py <nfiles>")
        sys.exit()

if __name__ == "__main__":
    main()


# setup.py file

from setuptools import find_packages, setup

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(
    name="lamdata_dspt6_rawalk",  # the name that you will install via pip
    version="1.0.1",
    author="Kush Rawal",
    author_email="kush-rawal@lambdastudents.com",
    description="2 helper utility functions",
    long_description=long_description,
    long_description_content_type="text/markdown",
    # required if using a md file for long desc
    # license="MIT",
    url="https://github.com/rawalk/Lambdata-rawalk",
    # keywords="",
    packages=find_packages()  # ["my_lambdata"]
)


from django.shortcuts import render
# from rest_framework import Response
from rest_framework.response import Response
from .models import Customer
from rest_framework.decorators import api_view
from .serializers import CustomerSerializer
# Create your views here.
from PIL import Image
from .S3Service import S3
from .imagga import Imagga_Request
from .tasks import check_request, send_mail_submit, send_mail_resubmit
from .Messages import *
from .mailgun import send_mail
from ipware import get_client_ip
@api_view(['GET'])
def ping(request):
    return Response({"PING": "PONG"})

@api_view(['POST'])
def submit_request(request):

    try:
        client_ip, _ = get_client_ip(request)
        customer_requests = Customer.objects.filter(national_id = request.data['national_id'])
        if len(customer_requests) != 0:
            for req in customer_requests:
                if req.state != "R":
                    return Response({"message": DUPLICATE}, status=400)

            customer_requests[0].state = "P"
            customer_requests[0].save()
            send_mail_resubmit(customer_requests[0])
            check_request.delay(customer_requests[0].email,
                                customer_requests[0].last_name,
                                customer_requests[0].img1.name.split('/')[-1],
                                customer_requests[0].img2.name.split('/')[-1])

            return Response({"message": RESUBMIT}, status=200)
        new_customer = Customer(email=request.data['email'],
                                last_name=request.data['last_name'],
                                national_id=request.data['national_id'],
                                img1=request.data['img1'],
                                img2=request.data['img2'],
                                user_ip=client_ip, state="P")
        new_customer.save()
        s3 = S3()

        s3.insert_object(new_customer.img1.name)
        s3.insert_object(new_customer.img2.name)

        send_mail_submit.delay(new_customer.email,
                               new_customer.last_name
                               )
        check_request.delay(new_customer.email,
                            new_customer.last_name,
                            new_customer.img1.name.split('/')[-1],
                            new_customer.img2.name.split('/')[-1]
                            )
        return Response({"message": SUBMIT}, status=200)
    except Exception as exc:
        new_customer.delete()
        return Response({"message": str(exc)}, status=500)

@api_view(['POST'])
def get_status(request):
    try:
        client_ip, _ = get_client_ip(request)

        national_id = request.data['national_id']
        customer = Customer.objects.get(national_id=national_id)
        if client_ip != customer.user_ip:
            return Response({"message": ACCESS_DENIED}, status=403)
        message = ''
        _s = customer.state
        if _s == "P":
            message = PENDING
        elif _s == "r":
            message = REJECTED
        else:
            message = APPROVED
        return Response({"details": message}, status=200)
    except Exception as exc:
        return Response({"message": str(exc)}, status=500)

@api_view(['GET'])
def testing(request):
    try:
        return Response({"message": "there is no test here"}, status=200)
    except Exception as exc:
        print(exc)
        return Response({"message": str(exc)}, status=500)

@api_view(['GET'])
def echo_ip(request):
    _ip, _ = get_client_ip(request)
    return Response({"Message": _ip})


class Solution:
    def numsSameConsecDiff(self, n: int, k: int) -> List[int]:
        pairs = []
        for d1 in range(1, 10):
            for d2 in range(0, 10):
                if abs(d1 - d2) == k:
                    pairs.append([str(d1), str(d2)])
        addOn = n - 2
        while addOn > 0:
            newPairs = []
            for p in pairs:
                dy = int(p[-1])
                for dx in range(0, 10):
                    if abs(dy - dx) == k:
                        newPairs.append(p + [str(dx)])
            pairs = newPairs
            addOn -= 1
        return list(map(lambda x: int(''.join(x)), pairs))

                    # return self.createNums(pairs, n - 2)

    # def createRes(self, pairs):
    #     res = []
    #     for p in pairs:
    #         res.append()

    # def createNums(self, pairs, addOn):
    #     res = []
    #     for p in pairs:
    #         l = str(p[0]) + str(p[1])
    #         count = addOn
    #         while count > 0:
    #             l += l[-2]
    #             count -= 1
    #         res.append(int(l))
    #     return res


import torch
from base.models.model_container import ModelContainer
from base.train.metrics import accuracy
from base.data.data_container import DataContainer
from base.train.common_trainer import CommonTrainer
import torch.nn as nn
import torch.optim as optim
import matplotlib.pyplot as plt
import torch.nn.functional as F
import random
from real_cnn_model.train.loss import SoftCrossEntropy, NegLogRatioLoss, KLDivLogits, InfoRadiusLoss
from data.imagenet import mutual_remove

class CNNTrainer(CommonTrainer):
    def __init__(self, cfg, model_container: ModelContainer, data_container: DataContainer, **kwargs):
        super(CNNTrainer, self).__init__(cfg, model_container, data_container)
        print(f'Initializing trainer {self.__class__.__name__}...')
        self.init_env()
        self.init_optimizer()
        self.init_scheduler()
        self.prep_train()
        self.loss_func = nn.CrossEntropyLoss()
        self.debug = getattr(self.cfg, 'debug', False)
        self.debug_input = getattr(self.cfg, 'debug_input', False)
        self.debug_labels = getattr(self.cfg, 'debug_labels', False)

    def init_optimizer(self, **kwargs):
        """
        Initialize self.optimizer using self.cfg.
        """

        # Choose optimizer
        model = self.model_container.models['model']
        opt_type = self.cfg.optimizer
        freeze = getattr(self.cfg, 'freeze', False) or getattr(self.cfg, 'train_classifier', False)

        if opt_type == 'SGD':
            print('Using SGD as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.SGD(model.parameters(), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
        elif opt_type == 'Adam':
            print('Using Adam as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.Adam(model.parameters(), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)

    def train(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']

        self.model_container.set_train(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])

        self.optimizer.zero_grad()

        pred = self.model_container.infer('model', input_data, False)

        loss = self.loss_func(pred, label)

        acc_1, acc_5 = accuracy(pred.cpu(), label.cpu(), topk=(1, min(5, pred.shape[-1])))
        loss.backward()
        self.optimizer.step()

        if self.debug:
            if self.debug_input:
                self.inspect_input(input_data)
            if self.debug_labels:
                self.inspect_labels(pred, label, acc_1)

        return loss.item(), acc_1, acc_5

    def test(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']

        self.model_container.set_eval(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])

        pred = self.model_container.infer('model', input_data, True)
        tot_num = len(pred)

        acc_1, acc_5 = accuracy(pred.cpu(), label.cpu(), topk=(1, min(5, pred.shape[-1])))

        num_correct = int(acc_1 * tot_num)

        if self.debug:
            if self.debug_input:
                self.inspect_input(input_data)
            if self.debug_labels:
                self.inspect_labels(pred, label, acc_1)

        return acc_1, acc_5, num_correct, tot_num

    def save_model(self, total_epoch, total_iter, total_val_iter, **kwargs):
        """
        Save model on self.exp_save_dir.

        Args:
            total_epoch: Current total epoch
            total_iter: Current total number of iterations
            total_val_iter: Current total number of validation iterations
        """

        save_mode = self.cfg.save_by
        multi_gpu = torch.cuda.device_count() >= 1 and self.cfg.parallel

        model = self.model_container.models['model']
        if multi_gpu:
            if save_mode == 'iter':
                if total_iter % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'iter': total_iter,
                        'state_dict': model.module.state_dict(),
                    }, save_dir)

            elif save_mode == 'epoch':
                if total_epoch % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'epoch': total_epoch,
                        'state_dict': model.module.state_dict(),
                    }, save_dir)
        else:
            if save_mode == 'iter':
                if total_iter % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'iter': total_iter,
                        'state_dict': model.state_dict(),
                    }, save_dir)

            elif save_mode == 'epoch':
                if total_epoch % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'epoch': total_epoch,
                        'state_dict': model.state_dict(),
                    }, save_dir)

    def prep_train(self):
        """
        Funcion called before training begins. Auxiliary function for initializng different training
        configurations such as model parallelism and weight freezing.
        """
        # Prepare for data parallelism
        self.model_container.load_saved()
        self.model_container.parallelize(['model'], self.devices)
        self.model_container.print_model_size(['model'])

    def inspect_labels(self, pred, label, acc_1):
        # Debugging utility for inspecting ground truth / predicted labels
        thres = getattr(self.cfg, 'acc_threshold', 0.1)
        if acc_1 < thres:
            unq, cnt = torch.unique(pred.argmax(-1), return_counts=True)
            print("GT label: " + self.data_container.dataset[self.cfg.mode].labels[label[0]] + f" {label[0]}")
            print("Most predicted: " + self.data_container.dataset[self.cfg.mode].labels[unq[cnt.argmax()]] + f" {unq[cnt.argmax()]}")

    def inspect_input(self, input_data):
        inspect_channel = getattr(self.cfg, 'inspect_channel', 0)
        inspect_index = getattr(self.cfg, 'inspect_index', 0)

        if inspect_index == 'random':
            inspect_index = random.randint(0, len(input_data) - 1)

        # Debugging utility for visualizing input data
        if type(inspect_channel) is int:
            tmp = input_data[inspect_index].permute(1, 2, 0)[:, :, inspect_channel]
            plt.imshow(255 * tmp.cpu().numpy())
            plt.show()

        if inspect_channel == 'all':
            fig = plt.figure(figsize=(50, 50))

            tmp = input_data[inspect_index].permute(1, 2, 0)
            num_channel = tmp.shape[-1]

            for i in range(num_channel):
                fig.add_subplot(num_channel // 2, 2, i + 1)
                plt.imshow(255 * tmp[..., i].cpu().numpy())

            plt.show()

        if inspect_channel == 'color':
            vis_data = input_data[inspect_index].permute(1, 2, 0)
            pos_ev = (vis_data[..., 0] != 0)
            neg_ev = (vis_data[..., 1] != 0)
            vis_img = torch.ones([vis_data.shape[0], vis_data.shape[1], 3], device=vis_data.device)
            vis_img[pos_ev] = torch.tensor([[1., 0, 0]], device=vis_data.device)
            vis_img[neg_ev] = torch.tensor([[0, 0, 1.]], device=vis_data.device)

            plt.imshow(vis_img.cpu())
            plt.show()

        if inspect_channel == 'color_chn':
            vis_data = input_data[inspect_index].permute(1, 2, 0)
            pos_ev = (vis_data[..., 0] != 0)
            neg_ev = (vis_data[..., 1] != 0)
            pos_img = torch.ones([vis_data.shape[0], vis_data.shape[1], 3], device=vis_data.device)
            pos_img[pos_ev] = torch.tensor([[1., 0, 0]], device=vis_data.device)
            neg_img = torch.ones([vis_data.shape[0], vis_data.shape[1], 3], device=vis_data.device)
            neg_img[neg_ev] = torch.tensor([[0, 0, 1.]], device=vis_data.device)

            fig = plt.figure(figsize=(50, 50))
            fig.add_subplot(1, 2, 1)
            plt.imshow(pos_img.cpu().numpy())
            fig.add_subplot(1, 2, 2)
            plt.imshow(neg_img.cpu().numpy())
            plt.show()

class EvTTACNNTrainer(CNNTrainer):
    def __init__(self, cfg, model_container: ModelContainer, data_container: DataContainer, **kwargs):
        super(EvTTACNNTrainer, self).__init__(cfg, model_container, data_container)
        print(f'Initializing trainer {self.__class__.__name__}...')
        self.loss_type = getattr(cfg, 'loss_type', 'entropy')
        if self.loss_type == 'entropy':
            self.loss_func = SoftCrossEntropy(1.)
        self.entropy_func = SoftCrossEntropy(1.)
        self.cos_loss_func = nn.CosineSimilarity(dim=-1)
        self.kl_loss_func = KLDivLogits()
        self.kl_distrib_loss_func = KLDivLogits(tgt_distrib=True)
        self.kl_log_loss_func = KLDivLogits(input_distrib=True)
        self.info_rad_loss_func = InfoRadiusLoss()
        if getattr(self.cfg, 'print_consistent', False):
            self.avg_consistent = 0.0
            self.counter = 0
        if getattr(self.cfg, 'num_train_samples', None) is not None:
            self.num_tot_train = len(self.data_container.dataset['train'])
            self.sample_counter = 0
        self.threshold_confidence = getattr(self.cfg, 'threshold_confidence', False)
        self.sigma_thres = getattr(cfg, 'sigma_thres', 0.25)

    def z_test_remove(self, input_data, augment_input_data = None):
        pos_count = (input_data[:, 0] != 0).flatten(1, 2).sum(-1)
        neg_count = (input_data[:, 1] != 0).flatten(1, 2).sum(-1)
        trans_val = self.data_container.dataset['train'].geary_trans(pos_count / neg_count)
        trans_mean = trans_val.mean()
        trans_std = trans_val.std()

        sigma = self.sigma_thres
        right_z_val = (trans_mean - (-sigma)) / (trans_std / (len(trans_val) ** (1 / 2)))
        right_p_val = self.data_container.dataset['train'].p_value(right_z_val)

        left_z_val = (trans_mean - (sigma)) / (trans_std / (len(trans_val) ** (1 / 2)))
        left_p_val = self.data_container.dataset['train'].p_value(left_z_val)

        if right_p_val < 0.1:
            input_data = mutual_remove(input_data, 0, 1)
        elif left_p_val > 0.9:
            input_data = mutual_remove(input_data, 1, 0)

        if augment_input_data is not None:
            return input_data, augment_input_data
        else:
            return input_data

    def train(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']
        augment_input_data = data_dict['augment_input_data']

        self.model_container.set_train(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])
            augment_input_data = augment_input_data.to(self.devices[0])

        if getattr(self.cfg, 'z_test_channel', False):
            input_data, augment_input_data = self.z_test_remove(input_data, augment_input_data)

        self.optimizer.zero_grad()

        total_pred = self.model_container.infer('model', torch.cat([input_data, augment_input_data], dim=0), False)

        label_pred = total_pred[:input_data.shape[0]]  # (B, L)
        label_augment_pred = total_pred[input_data.shape[0]:]  # (K * B, L)

        label_augment_pred = label_augment_pred.reshape(input_data.shape[0], getattr(self.cfg, 'num_sentry_augment', 3), -1)  # (B, K, L)
        label_idx = label_pred.argmax(-1).unsqueeze(-1)  # (B, 1)
        label_augment_idx = label_augment_pred.argmax(-1)  # (B, K)
        consistent = ((label_augment_idx == label_idx).sum(-1) / float(getattr(self.cfg, 'num_sentry_augment', 3)) > 0.5)  # (B, )

        total_pred = torch.cat([label_pred.unsqueeze(1), label_augment_pred], dim=1) # (B, K + 1, L)
        anchor_pred = label_pred

        # Optimize entropy of anchor_pred
        loss = self.loss_func(anchor_pred, anchor_pred, return_vector=True)
        loss[~consistent] *= getattr(self.cfg, 'sentry_inconsistent_fill', -1)
        loss = loss[loss != 0.].mean()

        # Set indices to compare with anchor
        other_idx = torch.stack([torch.arange(getattr(self.cfg, 'num_sentry_augment', 3)) + 1] * input_data.shape[0], dim=0)

        for idx in range(getattr(self.cfg, 'num_sentry_augment', 3)):
            tgt_pred_1 = total_pred[torch.arange(input_data.shape[0]), other_idx[:, idx], ...]
            tgt_pred_2 = anchor_pred

            if getattr(self.cfg, 'detach_tgt', False):  # Optionally detach gradients
                tgt_pred_1 = tgt_pred_1.detach()
                tgt_pred_2 = tgt_pred_2.detach()

            mutual_loss_type = getattr(self.cfg, 'mutual_loss_type', 'kl')
            loss += 0.5 * self.kl_loss_func(anchor_pred, tgt_pred_1) \
            + 0.5 * self.kl_loss_func(total_pred[torch.arange(input_data.shape[0]), other_idx[:, idx], ...], tgt_pred_2)

            # Target used for entropy loss
            entropy_tgt_pred = total_pred[torch.arange(input_data.shape[0]), other_idx[:, idx], ...]

            entropy_loss = self.loss_func(entropy_tgt_pred, entropy_tgt_pred, return_vector=True)
            entropy_loss[~consistent] *= getattr(self.cfg, 'sentry_inconsistent_fill', -1)
            loss += entropy_loss[entropy_loss != 0.].mean()

        tot_num = len(label_pred)

        loss.backward()
        self.optimizer.step()

        acc_1, acc_5 = accuracy(label_pred.cpu(), label.cpu(), topk=(1, min(5, label_pred.shape[-1])))
        num_correct = int(acc_1 * tot_num)

        return loss.item(), acc_1, acc_5, num_correct, tot_num

    def test(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']

        self.model_container.set_eval(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])

        if getattr(self.cfg, 'z_test_channel', False):
            input_data = self.z_test_remove(input_data)

        pred = self.model_container.infer('model', input_data, True)
        tot_num = len(pred)

        acc_1, acc_5 = accuracy(pred.cpu(), label.cpu(), topk=(1, min(5, pred.shape[-1])))

        num_correct = int(acc_1 * tot_num)

        if self.debug:
            if self.debug_input:
                self.inspect_input(input_data)
            if self.debug_labels:
                self.inspect_labels(pred, label, acc_1)

        return acc_1, acc_5, num_correct, tot_num

    def run_epoch(self):
        # Train
        print(f"This is {self.tracker.epoch}-th epoch.")

        self.tracker.init_epoch(mode='train')

        skip_train = getattr(self.cfg, 'skip_train', False)

        if not skip_train:
            self.train_epoch()

            print(f"Epoch {self.tracker.epoch} training loss = {self.tracker.total_train_loss.get_avg():.4f}")
            print(f"Total training accuracy = {(self.tracker.get_train_acc()):.4f}")

        # Validate
        print(f"Epoch {self.tracker.epoch} validation!")

        self.tracker.init_epoch(mode='val')
        self.validate_epoch()
        print(f"Total validation accuracy = {(self.tracker.get_val_acc()):.4f}")

        # Update scheduler
        if self.scheduler_type == 'reduce_plateau':
            self.scheduler.step(self.tracker.get_val_acc())
        elif self.scheduler_type is None:
            pass
        else:
            self.scheduler.step()

        # Save model by epoch (depends on cfg)
        if self.cfg.save_by == 'epoch':
            self.save_model(self.tracker.epoch, self.tracker.total_iter, self.tracker.total_val_iter)

    def train_batch(self, data_dict):
        self.tracker.init_batch(mode='train')
        self.tracker.start_infer_timing()

        loss, acc_1, acc_5, train_correct, train_num = self.train(data_dict)
        self.tracker.end_infer_timing()

        write_dict = {'training loss': loss, 'top 1 training accuracy': acc_1, 'top 5 training accuracy': acc_5, 'load time': self.tracker.load_time}
        print_dict = {'Iter': self.tracker.batch_idx, 'training loss': loss, 'top 1 training acc': acc_1, 'top 5 training acc': acc_5,
        'infer time': self.tracker.infer_time, 'load time': self.tracker.load_time}
        self.print_state(print_dict)
        self.write(self.tracker.total_iter, write_dict)

        self.tracker.total_train_loss.update(loss)
        self.tracker.total_train_correct.accumulate(train_correct)
        self.tracker.total_train_num.accumulate(train_num)

        if self.cfg.save_by == 'iter':
            self.save_model(self.tracker.epoch, self.tracker.total_iter, self.tracker.total_val_iter)

    def init_optimizer(self, **kwargs):
        """
        Initialize self.optimizer using self.cfg.
        """

        # Choose optimizer
        model = self.model_container.models['model']
        opt_type = self.cfg.optimizer
        freeze = getattr(self.cfg, 'freeze', False) or getattr(self.cfg, 'train_classifier', False)

        if opt_type == 'SGD':
            print('Using SGD as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.SGD(model.parameters(), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
        elif opt_type == 'Adam':
            print('Using Adam as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.Adam(model.parameters(), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)


import io
import zipfile

from pydantic import BaseModel, HttpUrl, ValidationError, field_validator

from web_resource_watchdog import Config

class CreateWebResource(BaseModel):
    """Pydantic schema for web resource model."""

    full_url: HttpUrl

class ZipFileValidator(BaseModel):
    """Pydantic schema for zip-file validation."""

    zip_file: bytes

    @field_validator("zip_file")
    def validate_zip_file(cls, value):
        """Validate the uploaded zip file.

        Parameters:
            value (bytes): The uploaded zip file as bytes.

        Returns:
            io.BytesIO: A BytesIO object representing
            the validated zip buffer.

        Raises:
            ValidationError: If the uploaded file is not a valid
            zip archive.
        """
        zip_buffer = io.BytesIO(value)
        try:
            with zipfile.ZipFile(zip_buffer, "r") as zip_ref:
                zip_ref.testzip()
            return zip_buffer
        except zipfile.BadZipFile:
            raise ValidationError(
                "Invalid zip file",
            )

def allowed_file(filename):
    """Check file extension in allowed extensions."""
    return ("." in filename) and (
        filename.rsplit(".", 1)[1].lower()
        in Config.ALLOWED_ARCHIVES_EXTENSIONS
    )


import adsk.core, adsk.fusion, traceback
import math

# Global variable used to maintain a reference to all event handlers.
handlers = []

activeDoc = None

# Called when the command is executed.  However, because this command
# is using the ExecutePreview event and is setting the isValidResult property
# to true, the results created in the preview will be used as the final
# results and the Execute will not be called.
class CutoutCommandExecuteHandler(adsk.core.CommandEventHandler):
    def __init__(self):
        super().__init__()
    def notify(self, args):
        try:
            # Get the inputs entered in the dialog.
            inputs = args.command.commandInputs
            result = getInput(inputs)

            # Draw the geometry.
            app = adsk.core.Application.get()
            drawGeometry(result[0], result[1], result[2], result[3])
        except:
            app = adsk.core.Application.get()
            ui = app.userInterface
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

class CutoutCommandExecutePreviewHandler(adsk.core.CommandEventHandler):
    def __init__(self):
        super().__init__()
    def notify(self, args):
        # Code to react to the event.
        try:
            cmdArgs = adsk.core.CommandEventArgs.cast(args)

            # Get the current value of inputs entered in the dialog.
            inputs = cmdArgs.command.commandInputs
            result = getInput(inputs)

            # Draw the preview geometry.
            drawGeometry(result[0], result[1], result[2], result[3])

            # Set this property indicating that the preview is a good
            # result and can be used as the final result when the command
            # is executed.
            cmdArgs.isValidResult = True
        except:
            app = adsk.core.Application.get()
            ui = app.userInterface
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

# Calculate the minimum distance between the input point and plane.
def MinDistPointToPlane(point, plane):
     temp = -(plane.origin.x * plane.normal.x + plane.origin.y * plane.normal.y + plane.origin.z * plane.normal.z)

     minDistPointToPlane = (plane.normal.x * point.x + plane.normal.y * point.y + plane.normal.z * point.z + temp) / math.sqrt(plane.normal.x ** 2 + plane.normal.y ** 2 + plane.normal.z ** 2)
     return minDistPointToPlane

class CutoutCommandInputChangedHandler(adsk.core.InputChangedEventHandler):
    def __init__(self):
        super().__init__()
    def notify(self, args):
        try:
            # Check to see if it's the plane input.
            if args.input.id == 'planeSelect':
                # Enable/disable the point input field depending on if the plane input has been specified.
                planeInput = args.input
                pointInput = args.firingEvent.sender.commandInputs.itemById('pointSelect')
                if planeInput.selectionCount == 1:
                    pointInput.isEnabled = True
                else:
                    pointInput.clearSelection()
                    pointInput.isEnabled = False
        except:
            app = adsk.core.Application.get()
            ui = app.userInterface
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

class CutoutCommandSelectionEventHandler(adsk.core.SelectionEventHandler):
    def __init__(self):
        super().__init__()
    def notify(self, args):
        try:
            # Check to see that selection of points is being done.
            if args.firingEvent.activeInput.id == 'pointSelect':
                # Get the selected plane.
                inputs = args.firingEvent.sender.commandInputs
                planeSelect = inputs.itemById('planeSelect')
                planeGeom = planeSelect.selection(0).entity.geometry

                # Get the geometry of the point being selected.
                sel = args.selection
                ent = sel.entity
                if ent.objectType == adsk.fusion.SketchPoint.classType():
                    pointGeom = ent.worldGeometry
                else:
                    pointGeom = ent.geometry

                # Validate that the point lies on the plane and set whether it is selectable or not.
                if math.fabs(MinDistPointToPlane(pointGeom, planeGeom)) < 0.00001:
                    args.isSelectable = True
                else:
                    args.isSelectable = False
        except:
            app = adsk.core.Application.get()
            ui = app.userInterface
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

# Event handler class.
class CutoutCommandCreatedHandler(adsk.core.CommandCreatedEventHandler):
    def __init__(self):
        super().__init__()
    def notify(self, args):
        try:
            app = adsk.core.Application.get()
            ui = app.userInterface

            global activeDoc
            activeDoc = app.activeDocument

            # Define the command dialog.
            cmd = adsk.core.Command.cast(args.command)
            cmdInputs = cmd.commandInputs

            # Create the selector for the plane.
            planeInput = cmdInputs.addSelectionInput('planeSelect', 'Select Plane', 'Select Plane')
            planeInput.addSelectionFilter('PlanarFaces')
            planeInput.addSelectionFilter('ConstructionPlanes')
            planeInput.setSelectionLimits(1,1)

            # Create the selector for the points.
            pointInput = cmdInputs.addSelectionInput('pointSelect', 'Select Points', 'Select Points')
            pointInput.addSelectionFilter('Vertices')
            pointInput.addSelectionFilter('ConstructionPoints')
            pointInput.addSelectionFilter('SketchPoints')
            pointInput.setSelectionLimits(1,0)
            pointInput.isEnabled = False

            # Create the list for types of shapes.
            shapeList = cmdInputs.addDropDownCommandInput('shapeList', 'Shape Type', adsk.core.DropDownStyles.LabeledIconDropDownStyle)
            shapeList.listItems.add('Square', True, 'Resources/Square', -1)
            shapeList.listItems.add('Circle', False, 'Resources/Circle', -1)
            shapeList.listItems.add('Pentagon', False, 'Resources/Pentagon', -1)

            # Create the slider input for the size.
            des = adsk.fusion.Design.cast(app.activeProduct)
            um = des.unitsManager
            oneUnit = um.convert(1, des.unitsManager.defaultLengthUnits, 'cm') * 4
            sizeSlider = cmdInputs.addFloatSliderCommandInput('sizeSlider', 'Size', des.unitsManager.defaultLengthUnits, oneUnit, oneUnit * 15, False)
            sizeSlider.valueOne = oneUnit * 4
            sizeSlider.spinStep = oneUnit/2

            # Connect to the execute event.
            onExecute = CutoutCommandExecuteHandler()
            cmd.execute.add(onExecute)
            handlers.append(onExecute)

            # Connect to the execute preview event.
            onExecutePreview = CutoutCommandExecutePreviewHandler()
            cmd.executePreview.add(onExecutePreview)
            handlers.append(onExecutePreview)

            # Connect to the input changed event.
            onInputChanged = CutoutCommandInputChangedHandler()
            cmd.inputChanged.add(onInputChanged)
            handlers.append(onInputChanged)

            # Connect to the selection event.
            onSelectionEvent = CutoutCommandSelectionEventHandler()
            cmd.selectionEvent.add(onSelectionEvent)
            handlers.append(onSelectionEvent)
        except:
            app = adsk.core.Application.get()
            ui = app.userInterface
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

# Gets the current values from the command dialog.
def getInput(inputs):
    try:
        for input in inputs:
            if input.id == 'planeSelect':
                planeEnt = input.selection(0).entity
            elif input.id == 'pointSelect':
                pointEnts = adsk.core.ObjectCollection.create()
                for i in range(0, input.selectionCount):
                    pointEnts.add(input.selection(i).entity)
            elif input.id == 'sizeSlider':
                size = input.valueOne
            elif input.id == 'shapeList':
                shape = input.selectedItem.name

        return (planeEnt, pointEnts, shape, size)
    except:
        app = adsk.core.Application.get()
        ui = app.userInterface
        ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

# Draws the shapes based on the input argument.
def drawGeometry(planeEnt, pointEnts, shape, size):
    try:
        # Get the design.
        app = adsk.core.Application.get()
        des = adsk.fusion.Design.cast(app.activeProduct)

        # Create a new sketch plane.
        sk = des.rootComponent.sketches.add(planeEnt)

        for pntEnt in pointEnts:
            # Project the point onto the sketch.
            skPnt = sk.project(pntEnt).item(0)

            if shape == 'Square':
                # Draw four lines to define a square.
                skLines = sk.sketchCurves.sketchLines
                line1 = skLines.addByTwoPoints(adsk.core.Point3D.create(skPnt.geometry.x - size/2, skPnt.geometry.y - size/2, 0), adsk.core.Point3D.create(skPnt.geometry.x + size/2, skPnt.geometry.y - size/2, 0))
                line2 = skLines.addByTwoPoints(line1.endSketchPoint, adsk.core.Point3D.create(skPnt.geometry.x + size/2, skPnt.geometry.y + size/2, 0))
                line3 = skLines.addByTwoPoints(line2.endSketchPoint, adsk.core.Point3D.create(skPnt.geometry.x - size/2, skPnt.geometry.y + size/2, 0))
                line4 = skLines.addByTwoPoints(line3.endSketchPoint, line1.startSketchPoint)
            elif shape == 'Circle':
                # Draw a circle.
                sk.sketchCurves.sketchCircles.addByCenterRadius(skPnt, size/2)
            elif shape == 'Pentagon':
                # Draw file lines to define a pentagon.
                skLines = sk.sketchCurves.sketchLines
                angle = math.pi/2
                halfSize = size/2
                x1 = halfSize * math.cos(angle)
                y1 = halfSize * math.sin(angle)

                angle += math.pi/2.5
                x2 = halfSize * math.cos(angle)
                y2 = halfSize * math.sin(angle)
                line1 = skLines.addByTwoPoints(adsk.core.Point3D.create(x1 + skPnt.geometry.x, y1 + skPnt.geometry.y, 0), adsk.core.Point3D.create(x2 + skPnt.geometry.x, y2 + skPnt.geometry.y, 0))

                angle += math.pi/2.5
                x = halfSize * math.cos(angle)
                y = halfSize * math.sin(angle)
                line2 = skLines.addByTwoPoints(line1.endSketchPoint, adsk.core.Point3D.create(x + skPnt.geometry.x, y + skPnt.geometry.y, 0))

                angle += math.pi/2.5
                x = halfSize * math.cos(angle)
                y = halfSize * math.sin(angle)
                line3 = skLines.addByTwoPoints(line2.endSketchPoint, adsk.core.Point3D.create(x + skPnt.geometry.x, y + skPnt.geometry.y, 0))

                angle += math.pi/2.5
                x = halfSize * math.cos(angle)
                y = halfSize * math.sin(angle)
                line4 = skLines.addByTwoPoints(line3.endSketchPoint, adsk.core.Point3D.create(x + skPnt.geometry.x, y + skPnt.geometry.y, 0))

                line5 = skLines.addByTwoPoints(line4.endSketchPoint, line1.startSketchPoint)

        # Find the inner profiles (only those with one loop).
        profiles = adsk.core.ObjectCollection.create()
        for prof in sk.profiles:
            if prof.profileLoops.count == 1:
                profiles.add(prof)

        # Create the extrude feature.
        input = des.rootComponent.features.extrudeFeatures.createInput(profiles, adsk.fusion.FeatureOperations.CutFeatureOperation)
        input.setDistanceExtent(True, adsk.core.ValueInput.createByReal(10))
        extrude = des.rootComponent.features.extrudeFeatures.add(input)
    except:
        app = adsk.core.Application.get()
        ui = app.userInterface
        ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

def run(context):
    ui = None
    try:
        app = adsk.core.Application.get()
        ui  = app.userInterface

        existingDef = ui.commandDefinitions.itemById('ekinsCutouts')
        if existingDef:
            existingDef.deleteMe()

        # Create the command definition.
        cutoutsCmdDef = ui.commandDefinitions.addButtonDefinition('ekinsCutouts', 'Cutout Shapes', 'Creates various shaped cutouts on the selected face at the specified locations.', './Resources/Cutouts')

        # Connect to the command created event.
        onCommandCreated = CutoutCommandCreatedHandler()
        cutoutsCmdDef.commandCreated.add(onCommandCreated)
        handlers.append(onCommandCreated)

        # Get the CREATE toolbar panel.
        createPanel = ui.allToolbarPanels.itemById('SolidCreatePanel')

        # Add the command below the Web command.
        createPanel.controls.addCommand(cutoutsCmdDef, 'FusionWebCommand', False)
    except:
        if ui:
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

def stop(context):
    ui = None
    try:
        app = adsk.core.Application.get()
        ui  = app.userInterface

        # Delete the control
        createPanel = ui.allToolbarPanels.itemById('SolidCreatePanel')
        cutoutsControl = createPanel.controls.itemById('ekinsCutouts')
        if cutoutsControl:
            cutoutsControl.deleteMe()

        # Delete the command definition.
        existingDef = ui.commandDefinitions.itemById('ekinsCutouts')
        if existingDef:
            existingDef.deleteMe()

    except:
        if ui:
            ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))


import os

def fixString(source):
    prepareStr = source.split("/")
    if len(prepareStr[1]) < 2:
        source = prepareStr[0] + "/" + "0" + prepareStr[1]
    return source

f = open("dataset.csv", "r")
lines = f.readlines()

arr = []
hashset = set()
hashset2 = set()
actualYear = ""
lastMonth = ""
fMonth = ""
string = ""
weekCounter = 0
fDay = 0

for i in range(len(lines)):
    # get the I'th line from the file and split it
    sliced = lines[i].split(",")

    # get current Year
    actualYear = sliced[0]

    if actualYear.split("/")[0] not in hashset:
        hashset2.clear()
    # save only the month
    hashset2.add(actualYear.split("/")[1])

    # save only the year
    hashset.add(actualYear.split("/")[0])

    # save first Month
    if arr == []:
        fDay = sliced[1]
        fMonth = actualYear

    # prepare array with the Data from file
    for rr in range(len(sliced)):
        if rr > 0 and rr < len(sliced) - 1:
            string += sliced[rr] + ","
        elif rr > 0:
            string += sliced[rr]
    # append the string
    arr.append(string)

    # go next week
    weekCounter += 1

    if actualYear == "2022/11":
        A = 5

    # get next Year
    if i < len(lines) - 1:
        nextYear = lines[i + 1].split(",")[0]
    else:
        nextYear = "EOF"

    # if reached the end of file or the nextYear has not been added to the set
    if (
        nextYear.split("/")[0] not in hashset
        or nextYear.split("/")[1] not in hashset2
        or nextYear == "EOF"
        or weekCounter == 7
    ):
        actualYear = (
            fixString(actualYear) + "/" + (fDay if len(fDay) == 2 else "0" + fDay)
        )
        endOfYear = (
            fixString(fMonth)
            + "/"
            + (sliced[1] if len(sliced[1]) == 2 else "0" + sliced[1])
        )

        # concatenate the string with new and old date
        fName = actualYear.replace("/", "") + "_" + endOfYear.replace("/", "")

        # open file and write to it
        with open(f"files\weeks\{fName}.csv", "w") as file:
            for item in arr:
                file.write(item)

        # nullify the array
        arr = []

        # nullify the weekCounter
        weekCounter = 0

    # nullify the string after the loop iteration (always)
    string = ""


# Written By: Nick Evans
# Purpose: Lightweight script to append timestamp to the beginning of .xisf image files to enable pixinsight blink tool to load images in the correct order.
# Usage: Either
# 1) place this script in the directory with your .xisf files you wish to rename, and run it or
# 2) run this script from commandline with argument to the path to where the files are. Example: $ python3 timestamp_files.py /c/images/

import time
import os
from pathlib import Path
import sys

## Get Path to process
if(sys.argv[1] != ""):
    # get path via command line argument
    path = Path(sys.argv[1])
else:
    # get path from where the script is ran from
    path = Path(os.getcwd())

### Insert path validation code / error catching here...

# Update user
print('------------------------------------')
print('Input path = ', path)
print('Detected %s files. Processing... ' %len(os.listdir(path)))
print('------------------------------------')

# Process Files
for x in os.listdir(path):
    if x.endswith(".xisf"): #could update with different file extension - perhaps via another argument
        f_path = os.path.join(path, x)

        # Obtaining the modified time (in seconds) of the file/folder
        t = os.path.getmtime(f_path)

        # Converting the time to an epoch string
        t_str = time.ctime(t)

        # Converting the string to a time object
        t_obj = time.strptime(t_str)

        # Transforming the time object to a timestamp to ISO 8601 format
        form_t = time.strftime("%Y-%m-%d %H:%M:%S", t_obj)

        # Modified Letter Colon " " (U+A789)
        form_t = form_t.replace(":", "꞉")

        # Renaming the filename to its timestamp
        os.rename(f_path, os.path.split(f_path)[0] + '/' + form_t +'_'+ x)
        print(os.path.split(f_path)[0] + '/' + form_t +'_'+ x) #+ os.path.splitext(f_path)[1]) # for the extension
print('------------------------------------')
print('Processing Complete.')


import argparse
from flask import Flask, jsonify, request
import socket
from pathlib import Path
from platform import system as OS

from syncFiles.syncFiles import check_sum

DEBUG = True
currentIP = socket.gethostbyname(socket.gethostname())

ap = argparse.ArgumentParser(description='Run a deamon that checks sums of sent files.',
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)

ap.add_argument('storage_folder_path',
                help='Path to the folder where the data is supposed to be moved.',
                type=Path)
ap.add_argument('--host',
                help="Host's IP.",
                default=currentIP)
ap.add_argument('--port',
                help='Port to listen to.',
                default=9001,
                type=int)
ap = ap.parse_args()
app  = Flask(__name__)

@app.route('/')
def index():
    return f'checkSummer operating.\nIP: {ap.host} PORT: {ap.port}\nDATA FOLDER:{ap.storage_folder_path}'

@app.route('/check', methods=['POST'])
def get_project_id():
    if request.data:
        print(request.data)
        print(ap.storage_folder_path)
        relative_path, check_sum_kwds = request.get_json()
        local_abs_file_path = ap.storage_folder_path/relative_path
        print(local_abs_file_path)
        cs = check_sum(local_abs_file_path, **check_sum_kwds)
        print(cs)
        return jsonify(cs)

if __name__ == '__main__':
    app.run(debug=DEBUG,
            host=ap.host,
            port=ap.port,
            threaded=True)