import torch as t
from torch import nn
from torch.utils.data import Dataset, DataLoader
from torch.optim import Adam

import numpy as np
import cv2
import os

batch_size = 16
img_size = 32
latent_dim = 64
channels = 1
lr = 0.00001 #0.0002
b1, b2 = 0.5, 0.999
n_epochs = 50
n_ch = 64  # Number of channels in initial convolution layers
sample_interval = 200  # Save a generated image every sample_interval number of batches

write_dir = '../images/gan_conv_test/'
data_dir = '../data/bub_single_24new/'
os.makedirs(write_dir, exist_ok=True)

Zeros = t.zeros((batch_size, 1, img_size, img_size))
Ones = t.ones((batch_size, 1, img_size, img_size))

class Data(Dataset):
    def __init__(self):
        self.data = [cv2.imread(data_dir + im, cv2.IMREAD_GRAYSCALE)//255 for im in os.listdir(data_dir) if '-0.png' in im]
        self.data = t.Tensor([im for im in self.data if im.shape == (img_size, img_size)])

        self.len = self.data.shape[0]

    def __len__(self):
        return self.len

    def __getitem__(self, index):
        return self.data[index]

dataset = Data()

dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

img_shape = (channels, img_size, img_size)

class Generator(nn.Module):
    # Takes as input a random tensor of size (batch_size, latent_dim) generated by noise()
    '''def __init__(self):
        super(Generator, self).__init__()

        self.lin = nn.Sequential(nn.Linear(latent_dim, n_ch * (img_size//2)**2))
        # Reshape to (N, n_ch, im_size//2, im_size//2)
        self.conv_block = nn.Sequential(
            nn.BatchNorm2d(n_ch),
            nn.Conv2d(n_ch, n_ch//2, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(n_ch//2),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(n_ch//2, n_ch//4, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(n_ch//4),
            nn.LeakyReLU(0.2, inplace=True),
            #nn.Upsample(scale_factor=2),
            nn.Conv2d(n_ch//4, channels, kernel_size=3, stride=1, padding=1),
            nn.Sigmoid()
            )

    def forward(self, im):
        im = self.lin(im)
        im = im.view(im.shape[0], n_ch, img_size//2, img_size//2)
        return self.conv_block(im)

    # (N, latent_dim) ->[linear] (N, n * 16**2) ->[reshape] (N, n, 16, 16) ->[conv] (N, n/2, 16, 16) ->[upsampling] (N, n/2, 32, 32) ->[conv] (N, n/4, 32, 32) ->[conv] (N, 1, 32, 32)'''

    def __init__(self):
        super(Generator, self).__init__()

        self.init_size = img_size//4
        self.linear = nn.Sequential(nn.Linear(latent_dim, n_ch * self.init_size**2))  #Outsize=(8x8xn) with n=128

        self.conv_blocks = nn.Sequential(
            nn.BatchNorm2d(n_ch),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(n_ch, n_ch, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(n_ch, 0.8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(n_ch, n_ch//2, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(n_ch//2, 0.8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(n_ch//2, channels, kernel_size=3, stride=1, padding=1),
            nn.Sigmoid()
            )

    def forward(self, noise):
        generated_im = self.linear(noise)
        generated_im = generated_im.view(generated_im.shape[0], n_ch, self.init_size, self.init_size)
        generated_im = self.conv_blocks(generated_im)
        #generated_im = t.where(generated_im > 0.5, Ones, generated_im)
        return generated_im

    # (N, latent_dim) ->[linear] (N, n * 8**2) ->[reshape] (N, n, 8, 8) ->[upsampling] (N, n, 16, 16) ->[conv] (N, n, 16, 16) ->[upsampling] (N, n, 32, 32) ->[conv] (N, n/2, 32, 32) ->[conv] (N, 1, 32, 32)

class Discriminator(nn.Module):
    # Takes as input a data image or image generated by Generator
    def __init__(self):
        super(Discriminator, self).__init__()

        self.model = nn.Sequential(
            nn.Linear(int(np.prod(img_shape)), 8*latent_dim),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(8*latent_dim, 4*latent_dim),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(4*latent_dim, 1),
            nn.Sigmoid(),
        )

    def forward(self, img):
        img_flat = img.view(img.size(0), -1)
        validity = self.model(img_flat)
        return validity

def generate_noise():
    return t.rand((batch_size, latent_dim))

# Loss function
adversarial_loss = nn.BCELoss()

G = Generator()
D = Discriminator()

# Optimizers
optimizer_G = Adam(G.parameters(), lr=lr, betas=(b1, b2))
optimizer_D = Adam(D.parameters(), lr=lr, betas=(b1, b2))

for epoch in range(n_epochs):
    for i, data_imgs in enumerate(dataloader):

        # Configure generator input
        data_shape = data_imgs.shape
        data_imgs = data_imgs.view(data_imgs.shape[0], channels, data_imgs.shape[1], data_imgs.shape[2])

        # Adversarial ground truths
        valid = t.ones((data_shape[0], 1), requires_grad=False)
        fake = t.zeros((data_shape[0], 1), requires_grad=False)

        # ----------------
        # Train Generator
        # ----------------

        optimizer_G.zero_grad()

        # Sample noise as generator input
        noise = generate_noise()

        # Generate a batch of images
        gen_imgs = G(noise)
        gen_imgs = gen_imgs[:data_shape[0]]

        # Loss measures generator's ability to fool the discriminator
        G_loss = adversarial_loss(D(gen_imgs), valid)

        G_loss.backward()
        optimizer_G.step()

        # --------------------
        # Train Discriminator
        # --------------------

        optimizer_D.zero_grad()

        # Measure discriminator's ability to classify real from generated samples
        real_loss = adversarial_loss(D(data_imgs), valid)
        fake_loss = adversarial_loss(D(gen_imgs.detach()), fake)
        D_loss = (real_loss + fake_loss) / 2

        D_loss.backward()
        optimizer_D.step()

        # ----------------
        # Log Progress
        # ----------------

        print(
            "[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]"
            % (epoch, n_epochs, i, len(dataloader), D_loss.item(), G_loss.item())
        )

        batches_done = epoch*len(dataloader) + i
        if batches_done % sample_interval == 0:
            os.makedirs(write_dir + str(batches_done), exist_ok=True)
            for j, gen_im in enumerate(gen_imgs):
                cv2.imwrite('%s%d/%d.png' % (write_dir, batches_done, j), gen_im[0].detach().numpy()*255)


from socket import *

host = 'Marcelo-Laptop'
port  = 9123

s = socket(AF_INET, SOCK_STREAM)

try: s.bind((host, port))
except error as e: print("ERROR:", e)

s.listen(1)
print('Aan het wachten voor een client.........')

conn, addr = s.accept()

print("Established Connection.\nThe client is:", addr[0]+":"+str(addr[1]))

while True:
    try:
        data = conn.recv(2048)
        print("Raspberry pi: ", data.decode("UTF-8"))
        bericht = data.decode('UTF-8')
        if bericht == 'Rotterdam: 3m':
            reply = 'rood led'
            conn.sendall(bytes(reply, "UTF-8"))

        elif bericht == 'Rotterdam: minder dan 3m':
            reply = 'groen led'
            conn.sendall(bytes(reply, "UTF-8"))
    except:
        break

print("Sorry something went wrong! You have lost connection to the client.:(")
conn.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

import random

def enemy(ancestry, gear):
    funcenemy = ancestry
    weapon = gear
    hp = random.randrange(0, 20)
    ac = random.randrange(0, 20)
    return [funcenemy, weapon, hp, ac]

def fight(tgt):
    print("You take a swing at the " + tgt[0] + ".")
    hit = random.randrange(0, 20)
    if hit > tgt[3]:
        print("You hit the " + tgt[0] + " for " + str(hit) + " damage!")
        tgt[2] = tgt[2] - hit
    else:
        print("You missed.")

def main():

    foe = enemy("troll", "great axe")
    print("You meet a " + foe[0] + " wielding a " + foe[1])
    print("Type the a key and then RETURN to attack.")

    while True:
        action = input()

        if action.lower() == "a":
            fight(foe)

        if foe[2] < 1:
            print("You killed your foe!")
        else:
            print("The " + foe[0] + " has " + str(foe[2]) + " HP remaining")

if __name__ == '__main__':
    main()


from flask import Flask, request, render_template
from flask_cors import cross_origin
import sklearn
import pickle
import pandas as pd

app = Flask(__name__)
model = pickle.load(open("oil_price.pkl","rb"))

@app.route("/")
@cross_origin()
def home():
    return render_template("petrol.html")

@app.route("/predict", methods = ["GET", "POST"])
@cross_origin()
def predict():
    if request.method == "POST":
        da=request.form["date"]
        day=int(pd.to_datetime(da, format="%Y-%m-%d").day)
        month=int(pd.to_datetime(da, format="%Y-%m-%d").month)
        year=int(pd.to_datetime(da, format="%Y-%m-%d").year)

        prediction=model.predict([[day,month,year]])
        output=round(prediction[0],2)
        return render_template('petrol.html',prediction_text="Petrol price is USD($). {}".format(output))
    return render_template("petrol.html")

if __name__ == "__main__":
    app.run(debug=True)


import os
import pickle
from pathlib import Path
import numpy as np

import click
import pandas as pd
from constants import TARGET_COLUMN

from feature_transform import ExperimentalTransformer

def load_object(filepath: str):
    with open(filepath, "rb") as f:
        obj = pickle.load(f)
    return obj

@click.command("make_prediction")
@click.option("--input-datafile")
@click.option("--path-to-model")
@click.option("--output-dir")
def make_prediction(input_datafile: str, path_to_model: str, output_dir: str):
    model = load_object(path_to_model)
    data = pd.read_csv(input_datafile)

    X = data.drop(columns=[TARGET_COLUMN], axis=1)

    model_dir = Path(path_to_model).parent
    transformer = ExperimentalTransformer(model_dir)
    transformer.load_from_file()

    X_transformed = transformer.transform(X)

    y_predict = model.predict(X_transformed)

    os.makedirs(output_dir, exist_ok=True)
    np.savetxt(os.path.join(output_dir, "predictions.csv"), y_predict, delimiter=',')

if __name__ == "__main__":
    make_prediction()


'''
This houses functions that are from an old version of inStrain; useful for testing
'''

import inStrain.controller

import inStrain.deprecated.deprecated_filter_reads

import inStrain.profile
import inStrain.profile.profile_utilities
import inStrain.profile.snv_utilities
# import inStrain.profileUtilities

P2C = {'A': 0, 'C': 1, 'T': 2, 'G': 3}  # base -> position

# THIS IS CC'S VERSION
def main(args):
    '''
    Main entry point
    '''

    # Set up Logger
    if not args.output:
        log = args.fasta.split("/")[-1].split(".")[0] + '.log'
    else:
        log = args.output + '.log'
    inStrain.controller.setup_logger(log)

    if args.filter_cutoff:
        if float(args.filter_cutoff) < 1:
            strain_pipeline(args, float(args.filter_cutoff))
        else:
            logging.error("Error: Strain level should be a number between 0 and 1.")
            sys.exit(1)
    else:
        strain_pipeline(args, 0.96)

def strain_pipeline(args, filter_cutoff):
    strains = SNVdata()

    # if args.testing:
    #     strains.testing = True

    if not args.output:
        strains.output = args.fasta.split("/")[-1].split(".")[0] + "_" + str(filter_cutoff)
    else:
        strains.output = args.output + "_" + str(filter_cutoff)

    logging.info("Running at resolution: (>" + str(filter_cutoff) + "%)")

    args.genes = None
    strains.get_scaffold_positions(args.genes, args.fasta)
    strains.run_strain_profiler(args.bam, min_cov=int(args.min_cov), min_freq=float(args.min_freq),
                                filter_cutoff=filter_cutoff)

    logging.info("\nCalculating linkage network...\n")
    strains.calc_linkage_network()

    logging.info("Calculating LD...\n")
    strains.calc_ld_all_sites(int(args.min_snp))
    strains.save()

class SNVdata:
    # The class "constructor" - It's actually an initializer
    def __init__(self):

        # Parameters
        self.fasta = ""
        self.bam = ""
        self.results = False
        self.output = None
        self.testing = False
        self.positions = []
        self.fasta_length = 0

        # Data structures
        self.snv_table = None  #
        self.read_to_snvs = None  #
        self.windows_to_snvs = None  #
        self.all_counts = None  # All counts of AGCT for each position
        self.snv_counts = None  # Counts of AGCT for each SNV position
        self.clonality_table = None  # Dict of clonality histograms (lists) by window
        self.snv_graph = None  # Weighted networkx graph that tracks SNVs( 100:A, 100:T are diff nodes) that are linked
        self.position_graph = None  # Unweighted networkx graph that tracks positions that are linked
        self.r2linkage_table = None  # Dict of r2 histograms (list of lists) by window

        # General statistics
        self.coverages = None
        self.total_positions = None
        self.total_snv_sites = None
        self.alpha_snvs = None

    def save(self, size=0):
        if self.results:
            # Generate tables
            if size == 0:
                self.read_to_snvs = None

            logging.info("printing to tables.")
            sample = self.bam.split("/")[-1].split(".bam")[0]

            genome = self.output
            logging.info(genome)
            self.snv_table.to_csv(genome + ".freq", sep='\t')  # , quoting=csv.QUOTE_NONE)
            self.clonality_table.to_csv(genome + ".clonal", sep='\t')  # , quoting=csv.QUOTE_NONE)
            self.r2linkage_table.to_csv(genome + ".linkage", sep='\t')  # , quoting=csv.QUOTE_NONE)
            self.mapping_info.to_csv(genome + ".readFiltering", sep='\t')  # ', quoting=csv.QUOTE_NONE)

            f = open(genome + ".data", 'wb')
            pickle.dump(self.__dict__, f, 2)
            f.close()
        else:
            logging.info("No data to save.")

    def load(self, name):
        self.__dict__.clear()
        f = open(name + ".data", 'rb')
        tmp_dict = pickle.load(f)
        f.close()

        self.__dict__.update(tmp_dict)

    def get_scaffold_positions(self, gene_list=None, fasta_file=None):
        ''' Returns a list of windows to record SNVs in'''
        if not fasta_file and not gene_list:
            #            logging.info("ERROR: REQUIRED TO SUPPLY FASTA or GENE FILE")
            sys.exit(1)

        if gene_list:
            f = open(gene_list)
            for line in f.readlines():
                self.positions.append([line.split(",")[1], int(line.split(",")[2]), int(line.split(",")[3])])
            f.close()
        else:
            for rec in SeqIO.parse(fasta_file, "fasta"):
                self.fasta_length += len(rec.seq)
                start = 0
                while True:
                    chunk = start + 15000
                    if chunk <= len(rec.seq):
                        self.positions.append([str(rec.id), start, start + 10000])
                    else:
                        self.positions.append([str(rec.id), start, len(rec.seq)])
                        break
                    start += 10000
                    start += 1

        self.fasta = fasta_file

    def get_snps_per_gene(self, gene_file=None):
        pass

    def calc_linkage_network(self):
        ''' Calculates the SNV linkage network - saves it as a dictionary of edges in self.snv_net.
        Writes it out to a file, genome.net for reading in through other programs like node2vec
        '''
        G = nx.Graph()  # graph between SNPs  (100:A and 100:C are different nodes)
        G_pos = nx.Graph()  # graph between positions; unweighted

        logging.info("Calculating SNV linkage network...")
        for read in self.read_to_snvs:
            snvs = self.read_to_snvs[read]
            for pair in itertools.combinations(snvs, 2):
                if G.has_edge(pair[0], pair[1]):
                    G[pair[0]][pair[1]]['weight'] += 1
                else:
                    G.add_edge(pair[0], pair[1], weight=1)
                    if not G_pos.has_edge(pair[0].split(":")[0], pair[1].split(":")[0]):
                        G_pos.add_edge(pair[0].split(":")[0], pair[1].split(":")[0])

        logging.info(str("Of " + str(self.total_snv_sites) + " SNP-sites, there were " + str(
            len(G_pos)) + " SNPs that could be linked to at least one other SNP."))
        logging.info("The average SNP was linked to " + str(
            int(np.mean([int(x[1]) for x in list(G_pos.degree())]))) + " other SNPs.")
        self.snv_graph = G
        self.position_graph = G_pos

    def calc_ld(self, snp_a, snp_b, min_snp, report=False):
        '''
        A function that calculates the LD between two SNPs
        '''

        # distance between snps
        distance = abs(int(snp_a.split("_")[-1]) - int(snp_b.split("_")[-1]))

        # calculate allele frequencies
        allele_A = major_allele(snp_a, self.snv_counts[snp_a])  # get major allele
        allele_a = minor_allele(snp_a, self.snv_counts[snp_a])  # get minor allele
        freq_A = float(allele_A[1]) / (allele_A[1] + allele_a[1])
        freq_a = float(allele_a[1]) / (allele_A[1] + allele_a[1])

        # get snp B frequencies
        allele_B = major_allele(snp_b, self.snv_counts[snp_b])
        allele_b = minor_allele(snp_b, self.snv_counts[snp_b])
        freq_B = float(allele_B[1]) / (allele_B[1] + allele_b[1])
        freq_b = float(allele_b[1]) / (allele_B[1] + allele_b[1])

        # Get frequencies of linkages
        countAB, countAb, countaB, countab = 0, 0, 0, 0
        if self.snv_graph.has_edge(allele_A[0], allele_B[0]):
            countAB = self.snv_graph[allele_A[0]][allele_B[0]]['weight']
        if self.snv_graph.has_edge(allele_A[0], allele_b[0]):
            countAb = self.snv_graph[allele_A[0]][allele_b[0]]['weight']
        if self.snv_graph.has_edge(allele_a[0], allele_B[0]):
            countaB = self.snv_graph[allele_a[0]][allele_B[0]]['weight']
        if self.snv_graph.has_edge(allele_a[0], allele_b[0]):
            countab = self.snv_graph[allele_a[0]][allele_b[0]]['weight']

        total = countAB + countAb + countaB + countab

        if report:
            print("total {0}".format(total))
            print([countAB, countAb, countaB, countab])

        # Requires at least min_snp
        if total > min_snp:

            linkage_points_x = []
            linkage_points_y = []
            for point in range(0, countAB):
                linkage_points_x.append(1)
                linkage_points_y.append(1)
            for point in range(0, countAb):
                linkage_points_x.append(1)
                linkage_points_y.append(0)
            for point in range(0, countaB):
                linkage_points_x.append(0)
                linkage_points_y.append(1)
            for point in range(0, countab):
                linkage_points_x.append(0)
                linkage_points_y.append(0)

            freq_AB = float(countAB) / total
            freq_Ab = float(countAb) / total
            freq_aB = float(countaB) / total
            freq_ab = float(countab) / total

            freq_A = freq_AB + freq_Ab
            freq_a = freq_ab + freq_aB
            freq_B = freq_AB + freq_aB
            freq_b = freq_ab + freq_Ab

            linkD = freq_AB - freq_A * freq_B

            if freq_a == 0 or freq_A == 0 or freq_B == 0 or freq_b == 0:
                r2_book = np.nan
            else:
                r2_book = linkD * linkD / (freq_A * freq_a * freq_B * freq_b)

            linkd = freq_ab - freq_a * freq_b
            # calc D-prime
            if (linkd < 0):
                denom = max([(-freq_A * freq_B), (-freq_a * freq_b)])
                d_prime = linkd / denom

            elif (linkD > 0):
                denom = min([(freq_A * freq_b), (freq_a * freq_B)])
                d_prime = linkd / denom
            else:
                d_prime = 0

            # r2 = stats.pearsonr(linkage_points_x, linkage_points_y)[0]
            # r2 = r2 * r2
            # logging.info(r2)
            # logging.info(r2_book)
            r2 = r2_book
            return ([distance, r2, linkD, linkd, r2_book, total, countAB, countAb, countaB, countab, allele_A, allele_a,
                     allele_B, allele_b])
            # else:
            #     logging.info("nan")
            # else:
        else:
            return False

    def calc_ld_all_sites(self, min_snp):
        '''
        Calculates Linkage Disequilibrium for all SNVs in a window.
        '''
        r2_total = {}
        sigma_total = {}
        logging.info("Calculating Linkage Disequilibrium")

        for window in self.positions:
            r2_spectrum = []
            sigma_spectrum = []

            window_name = window[0] + ":" + str(window[1]) + ":" + str(window[2])
            window_snvs = self.windows_to_snvs[window_name]

            for edge in self.position_graph.edges(window_snvs):
                snp_a = edge[0]
                snp_b = edge[1]

                ld_result = self.calc_ld(snp_a, snp_b, min_snp)
                if ld_result:
                    r2_spectrum.append(ld_result)

            r2_total[window_name] = r2_spectrum

        # Create r2 linkage table
        r2linkage_table = defaultdict(list)

        for window in r2_total:
            for datum in r2_total[window]:
                r2linkage_table['Window'].append(window)
                r2linkage_table['Distance'].append(datum[0])
                r2linkage_table['r2'].append(datum[1])
                r2linkage_table['linkD'].append(datum[2])
                # r2linkage_table['linkd'].append(datum[3])
                # r2linkage_table['r2_book'].append(datum[4])
                r2linkage_table['total'].append(datum[5])
                r2linkage_table['count_AB'].append(datum[6])
                r2linkage_table['count_Ab'].append(datum[7])
                r2linkage_table['count_aB'].append(datum[8])
                r2linkage_table['count_ab'].append(datum[9])

                r2linkage_table['total_A'].append(datum[10])
                r2linkage_table['total_a'].append(datum[11])
                r2linkage_table['total_B'].append(datum[12])
                r2linkage_table['total_b'].append(datum[13])

        self.r2linkage_table = pd.DataFrame(r2linkage_table)

    def run_strain_profiler(self, bam, min_cov=5, min_freq=0.05, filter_cutoff=0):
        '''
        Main class for finding SNVs and generating data profile for a genome.
        '''
        minimum_mapq = 2
        P2C = {'A': 0, 'C': 1, 'T': 2, 'G': 3}
        C2P = {0: 'A', 1: 'C', 2: 'T', 3: 'G'}

        # Set up variables
        alpha_snvs = 0
        total_positions = 0
        total_snv_sites = 0

        raw_counts_data = defaultdict(dict)  # Set up SNP table
        read_to_snvs = defaultdict(list)  # read name -> position : variant base
        snvs_frequencies = defaultdict(int)
        clonality_by_window = defaultdict(list)  # window -> [position, clonality]
        windows_to_snvs = defaultdict(list)  # window -> position

        all_counts = {}  # position -> coverage
        snv_counts = {}  # position -> [A, C, T, G]
        coverages = {}  # position -> coverage

        # read null model
        null_loc = os.path.dirname(__file__) + '/../helper_files/NullModel.txt'
        null_snp_model = inStrain.profile.snv_utilities.generate_snp_model(null_loc)

        ## Start reading BAM
        samfile = pysam.AlignmentFile(bam)
        sample = bam.split("/")[-1].split(".bam")[0]

        if self.testing:
            self.positions = self.positions[0:10]

        # Call read filtering function
        subset_reads, Rdb = inStrain.deprecated.deprecated_filter_reads.filter_reads(bam, self.positions, self.fasta_length,
                                                                          filter_cutoff, 3, 50, 2)

        # Save the read filtering report
        self.mapping_info = Rdb

        ### START SNP FINDING

        ## Start looping through each region
        for gene in tqdm(self.positions, desc='Finding SNVs ...'):
            scaff = gene[0]
            window = gene[0] + ":" + str(gene[1]) + ":" + str(gene[2])
            for pileupcolumn in samfile.pileup(scaff, truncate=True, max_depth=100000, stepper='nofilter',
                                               compute_baq=True, ignore_orphans=True, ignore_overlaps=True,
                                               min_base_quality=30):
                # for pileupcolumn in samfile.pileup(scaff, gene[1], gene[2], truncate = True, max_depth=100000, stepper = 'nofilter', compute_baq= True, ignore_orphans = True, ignore_overlaps = True,  min_base_quality = 30):
                ## Step 1: Are there any reads at this position?

                position = scaff + "_" + str(pileupcolumn.pos)
                counts = _get_base_counts(pileupcolumn, filtered_reads=subset_reads)

                consensus = False
                # Yes there were reads at this position
                if counts:
                    all_counts[position] = counts

                    if sum(counts) >= min_cov:
                        total_positions += 1
                        pos_clonality = inStrain.profile.snv_utilities.calculate_clonality(counts)
                        clonality_by_window[window].append([position, pos_clonality])
                        consensus = call_snv_site_old(counts, null_snp_model, min_cov=min_cov, min_freq=min_freq)
                        coverages[position] = sum(counts)

                # SEARCH HERE FOR DEBUG
                # if position == "N5_271_010G1_scaffold_114_990":
                #     print(counts)
                #     #counts2 = _get_base_counts_(pileupcolumn, filtered_reads = subset_reads)
                #     reads = []
                #     for pileupread in pileupcolumn.pileups:
                #         if not pileupread.is_del and not pileupread.is_refskip:
                #             if pileupread.alignment.query_name in subset_reads:
                #                 read_name = pileupread.alignment.query_name
                #                 reads.append((read_name, pileupread.alignment.query_sequence[pileupread.query_position]))
                #
                #     print(len(reads))
                #     print('\n'.join([str(r) for r in reads]))
                #     print(consensus)

                ## Strep 2: Is there an SNV at this position?
                if consensus:
                    # min_snp for this site
                    local_min_snp = null_snp_model[sum(counts)]
                    # there's an SNV at this site
                    total_snv_sites += 1
                    windows_to_snvs[window].append(position)
                    snv_counts[position] = counts

                    # Get reads to snvs
                    for pileupread in pileupcolumn.pileups:
                        read_name = pileupread.alignment.query_name
                        if not pileupread.is_del and not pileupread.is_refskip:
                            if read_name in subset_reads:
                                try:
                                    val = pileupread.alignment.query_sequence[pileupread.query_position]
                                    # if value is not the consensus value
                                    if counts[P2C[val]] >= local_min_snp and float(counts[P2C[val]]) / sum(
                                            counts) >= min_freq:
                                        # this is a variant read!
                                        read_to_snvs[read_name].append(position + ":" + val)
                                        if val != consensus:
                                            alpha_snvs += 1
                                except KeyError:  # This would be like an N or something not A/C/T/G
                                    pass

                    # Add to frequencies
                    nucl_count = 0
                    for nucl in counts:
                        if nucl >= local_min_snp:
                            freq = float(nucl) / float(sum(counts))
                            if freq >= min_freq:
                                snp = position + ":" + C2P[nucl_count]
                                snvs_frequencies[snp] = [freq, window]

                        nucl_count += 1

        #### SNP FINDING COMPLETE
        # Create SNV frequency table
        snv_table = defaultdict(list)
        for snv in snvs_frequencies:
            snv_table['SNV'].append(snv)
            snv_table['freq'].append(snvs_frequencies[snv][0])
            snv_table['Window'].append(snvs_frequencies[snv][1])
        snv_table = pd.DataFrame(snv_table)

        # Create clonality table
        clonality_table = defaultdict(list)
        for window in clonality_by_window:
            for position_pair in clonality_by_window[window]:
                clonality_table['scaffold'].append(window.split(":")[0])
                clonality_table['window_name'].append(window)
                clonality_table['position'].append(position_pair[0])
                clonality_table['clonality'].append(position_pair[1])
                clonality_table['coverage'].append(coverages[position_pair[0]])

        clonality_table_final = pd.DataFrame(clonality_table)

        # Final statistics
        logging.debug("Total SNVs-sites: " + str(total_snv_sites))
        logging.debug("Total SNV-bases: " + str(alpha_snvs))
        logging.debug("Mean clonality: " + str(
            float(sum(clonality_table['clonality'])) / float(len(clonality_table['clonality']))))
        logging.debug("Total sites above min-coverage: " + str(total_positions))
        logging.debug("Mean coverage: " + str(float(sum(coverages.values())) / len(coverages)))
        logging.debug("Total number of bases: " + str(sum(coverages.values())))

        self.coverages = coverages
        self.alpha_snvs = alpha_snvs
        self.total_snv_sites = total_snv_sites
        self.clonality_table = clonality_table_final
        self.snv_table = snv_table
        self.read_to_snvs = read_to_snvs
        self.total_positions = total_positions
        self.windows_to_snvs = windows_to_snvs
        self.all_counts = all_counts
        self.snv_counts = snv_counts

        self.results = True

def _get_base_counts(pileupcolumn, filtered_reads):
    '''
    From a pileupcolumn object, return a list with the counts of [A, C, T, G]
    '''
    P2C = {'A': 0, 'C': 1, 'T': 2, 'G': 3}
    C2P = {0: 'A', 1: 'C', 2: 'T', 3: 'G'}

    counts = [0, 0, 0, 0]
    empty = [0, 0, 0, 0]

    for pileupread in pileupcolumn.pileups:
        # logging.info(pileupread.)

        if not pileupread.is_del and not pileupread.is_refskip:
            read_name = pileupread.alignment.query_name
            if read_name in filtered_reads:
                # if pileupread.indel != 0:
                #     logging.info(pileupread.alignment.get_aligned_pairs())
                try:
                    counts[P2C[pileupread.alignment.query_sequence[pileupread.query_position]]] += 1
                except KeyError:  # This would be like an N or something not A/C/T/G
                    pass
    if counts != empty:
        return counts
    else:
        return False

def call_snv_site_old(counts, null_model, min_cov=5, min_freq=0.05):
    '''
    Determines whether a site has a variant based on its nucleotide count frequencies.
    '''
    P2C = {'A': 0, 'C': 1, 'T': 2, 'G': 3}
    C2P = {0: 'A', 1: 'C', 2: 'T', 3: 'G'}
    total = sum(counts)

    if total >= min_cov:
        i = 0
        for c in counts:
            if c >= null_model[total] and float(c) / total >= min_freq:
                i += 1
        if i > 1:
            return C2P[counts.index(max(counts))]
    else:
        return False

def major_allele(snv, counts):
    d = {'A': counts[0], 'C': counts[1], 'T': counts[2], 'G': counts[3]}
    nucl = sorted(d, key=d.get, reverse=True)[0]
    return [snv + ":" + nucl, d[nucl]]

def major_allele2(counts):
    d = {'A': counts[0], 'C': counts[1], 'T': counts[2], 'G': counts[3]}
    return sorted(d, key=d.get, reverse=True)[0]

def minor_allele(snv, counts):
    d = {'A': counts[0], 'C': counts[1], 'T': counts[2], 'G': counts[3]}
    nucl = sorted(d, key=d.get, reverse=True)[1]
    return [snv + ":" + nucl, d[nucl]]

def minor_allele2(counts):
    d = {'A': counts[0], 'C': counts[1], 'T': counts[2], 'G': counts[3]}
    return sorted(d, key=d.get, reverse=True)[1]

'''
           __..--''``---....___   _..._    __
 /// //_.-'    .-/";  `        ``<._  ``.''_ `. / // /
///_.-' _..--.'_                         `( ) ) // //
/ (_..-' // (< _     ;_..__               ; `' / ///
 / // // //  `-._,_)' // / ``--...____..-' /// / //

 Below this is super deprecated (used nowhere)
'''

# !/usr/bin/env python

'''
Possible names for this program:
* strainRep - not a legit acronym
* blockStrain - not a legit acronym
* destrain - not a legit acronym
* instrain - not a legit acronym
* metaIPD - metagenome interference of population structure - a legt acronym
'''

# Get the version
from inStrain._version import __version__

# Import
import os
import sys
import math
import pysam
import pickle
import logging
import argparse
import itertools
import numpy as np
import pandas as pd
import networkx as nx
from tqdm import tqdm
from Bio import SeqIO
from collections import defaultdict

## local imports
import inStrain.filter_reads

def entropy2(counts):
    probs = []
    total = sum(counts)
    for c in counts:
        probs.append(float(c) / total)

    ent = 0
    for i in probs:
        if i != 0:
            ent -= i * math.log(i, math.e)
    return ent

def getopts(argv):
    opts = {}  # Empty dictionary to store key-value pairs.
    while argv:  # While there are arguments left to parse...
        if argv[0][0] == '-':  # Found a "-name value" pair.
            opts[argv[0]] = argv[1]  # Add key and value to the dictionary.
        argv = argv[1:]  # Reduce the argument list by copying it starting from index 1.
    return opts

def iterate_windows(Wdb, snv_table, mm):
    '''
    Iterate windows
    '''
    Sdb = snv_table[snv_table['mm'] >= mm].sort_values('mm') \
        .drop_duplicates(subset=['scaffold', 'position'], keep='last') \
        .sort_index().drop(columns=['mm'])
    Sdb = Sdb.set_index('position', drop=False)

    for i, row in Wdb.iterrows():
        window = (int(row['start']), int(row['end']))
        sdb = Sdb[(Sdb['position'] >= window[0]) & (Sdb['position'] <= window[1])]
        snvs = list(sdb['position'].tolist())
        if len(snvs) > 0:
            yield snvs, window, sdb

def _get_series(s1, mm, lms1):
    # already a series
    if type(s1) == pd.Series:
        if s1['mm'] >= mm:
            return s1
        else:
            return lms1
    else:
        try:
            return s1.loc[mm]
        except:
            return lms1

def _update_counts(counts, ts1, ts2, mm):
    for b in ['A', 'C', 'T', 'G']:
        if len(ts1) > 0:
            counts[0][P2C[b]] += ts1[b]
        if len(ts2) > 0:
            counts[1][P2C[b]] += ts2[b]

def calc_SNV_linkage_network(read_to_snvs):
    '''
    Calculates the SNV linkage network

    Arguments:
        read_to_snvs = mm -> read -> SNVs

    saves it as a dictionary of edges in self.snv_net.
    Writes it out to a file, genome.net for reading in through other programs like node2vec
    '''
    mm_to_snv_graph = {}
    mm_to_position_graph = {}

    # logging.info("Calculating SNV linkage networks...")
    read2snvs = {}
    for mm, tread2snvs in read_to_snvs.items():
        G = nx.Graph()  # graph between SNPs  (100:A and 100:C are different nodes)
        G_pos = nx.Graph()  # graph between positions; unweighted

        read2snvs = {**read2snvs, **tread2snvs}
        for read in read2snvs:
            snvs = read2snvs[read]
            for pair in itertools.combinations(snvs, 2):
                if G.has_edge(pair[0], pair[1]):
                    G[pair[0]][pair[1]]['weight'] += 1
                else:
                    G.add_edge(pair[0], pair[1], weight=1)
                    if not G_pos.has_edge(int(pair[0].split(":")[0]), int(pair[1].split(":")[0])):
                        G_pos.add_edge(int(pair[0].split(":")[0]), int(pair[1].split(":")[0]))

        mm_to_snv_graph[mm] = G
        mm_to_position_graph[mm] = G_pos

    return mm_to_snv_graph, mm_to_position_graph

def _merge_tables_special(Cdb, Adb):
    FIX_COLS = ['ANI', 'SNPs', 'breadth_minCov']

    # Handle edge-case where there are no SNPs
    if len(Adb) == 0:
        for c in FIX_COLS:
            Cdb[c] = 0
        return Cdb

    # Make sure anything with a coverage has a SNP
    assert len(set(Adb['mm']) - set(Cdb['mm'])) == 0

    # Do initial merge
    Cdb = pd.merge(Cdb, Adb, on=['scaffold', 'mm'], how='outer')
    Cdb = Cdb.sort_values(['scaffold', 'mm'])
    Cdb = Cdb.reset_index(drop=True)

    # Run up NaN values. For example, in the cases
    Fdb = run_up_NaN(Cdb, FIX_COLS, on='scaffold')

    # Might as well adjust some datatypes
    pass

    return Fdb

def run_up_NaN(odb, cols, on='scaffold'):
    '''
    Take NaN values and fill them with the column above.
    For example, if you have mm of [0,1,2,3], and breadth of [.9, .92, NaN, .94],
    change the breadth to [.9, .92, .92, .94]
    If you have [0,1,2,3] and [NaN, NaN, .92, .94], it'll change it to:
    [0, 0, .92, .94]
    NOTE: Must be sorted / indexed in the order that you want run up
    Args:
        odb: original dataframe
        cols: columns to do this filling on
        on: the "key" of the dataframe. If this is all NaN, fill with 0s
    Returns:
        DataFrame: new dataframe
    '''
    Fdb = odb.copy()
    for scaff, db in odb.groupby(on):
        # handle edge case where all are NaN
        if len(db[cols[0]].dropna()) == 0:
            for i, row in db.iterrows():
                for col in cols:
                    Fdb.at[i, col] = 0
            continue

        # do the actual run-up
        top = True
        for i, row in db.iterrows():
            # hangle edge case where top values are NaN
            if top & np.isnan(row[cols[0]]):
                for col in cols:
                    Fdb.at[i, col] = 0
                continue
            else:
                top = False

            # The normal run-up case
            if np.isnan(row['ANI']):
                for col in cols:
                    Fdb.at[i, col] = Fdb.at[i - 1, col]

    return Fdb

def _zeros(mLen=1):
    return np.zeros(mLen, dtype=int)

def _clonT_to_clons(clonT, maxMM):
    '''
    Return the max clonality at each position
    '''
    counts = None
    for mm, count in [(mm, count) for mm, count in clonT.items() if mm <= maxMM]:
        if counts is None:
            counts = count
        else:
            counts = np.maximum(counts, count)

    if counts is None:
        return np.full(1, fill_value=np.nan)

    else:
        return counts

def make_ANI_table(snpsCounted, basesCounted, lengt, scaff, covT,
                   min_cov, Wdb):
    '''
    Fill in the SNP table with the SNPs and breadth_minCov for each scaffold and mm
    '''
    table = defaultdict(list)
    # fill in all SNP information
    for mm in sorted(list(covT.keys())):
        covs = inStrain.profile.profile_utilities.mm_counts_to_counts(covT, mm)
        if covs == [0, 0, 0, 0]:
            counted_basesO = 0
        else:
            zeros = (covs < min_cov).sum()
            counted_basesO = lengt - zeros

        counted_snps = _calc_counted_bases(snpsCounted, mm)
        counted_bases = _calc_counted_bases(basesCounted, mm)

        table['scaffold'].append(scaff)
        table['mm'].append(mm)
        table['SNPs'].append(counted_snps)
        # table['breadth_minCov'].append(counted_bases / lengt)
        if counted_bases == 0:
            table['ANI'].append(0)
        else:
            table['ANI'].append((counted_bases - counted_snps) / counted_bases)

    return pd.DataFrame(table)

def parse_arguments(args):
    parser = argparse.ArgumentParser(description="inStrain version {0}".format(__version__),
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # Required positional arguments
    parser.add_argument("bam", help="Sorted .bam file")
    parser.add_argument("fasta", help="Fasta file the bam is mapped to")

    # Optional arguments
    parser.add_argument("-o", "--output", action="store", default='inStrain', \
                        help='Output prefix')
    parser.add_argument("-p", "--processes", action="store", default=6, type=int, \
                        help='Threads to use for multiprocessing')
    parser.add_argument("-c", "--min_cov", action="store", default=5, \
                        help='Minimum SNV coverage')
    parser.add_argument("-s", "--min_snp", action="store", default=20, \
                        help='Absolute minimum number of reads connecting two SNPs to calculate LD between them.')
    parser.add_argument("-f", "--min_freq", action="store", default=0.05, \
                        help='Minimum SNP frequency to confirm a SNV (both this AND the 0.  001 percent FDR snp count cutoff must be true).')
    parser.add_argument("-fdr", "--fdr", action="store", default=1e-6, type=float, \
                        help='SNP false discovery rate- based on simulation data with a 0.1 percent error rate (Q30)')
    parser.add_argument("--min_scaffold_reads", action="store", default=0, type=int, \
                        help='Minimum number of reads mapping to a scaffold to proceed with profiling it')
    parser.add_argument("--scaffolds_to_profile", action="store", \
                        help='Path to a file containing a list of scaffolds to profile- if provided will ONLY profile those scaffolds')

    # Read filtering cutoffs
    parser.add_argument("-l", "--filter_cutoff", action="store", default=0.95, type=float, \
                        help='Minimum percent identity of read pairs to consensus to use the reads. Must be >, not >=')
    parser.add_argument("--min_mapq", action="store", default=-1, type=int, \
                        help='Minimum mapq score of EITHER read in a pair to use that pair. Must be >, not >=')
    parser.add_argument("--max_insert_relative", action="store", default=3, type=float, \
                        help='Multiplier to determine maximum insert size between two reads - default is to use 3x median insert size. Must be >, not >=')
    parser.add_argument("--min_insert", action="store", default=50, type=int, \
                        help='Minimum insert size between two reads - default is 50 bp. If two reads are 50bp each and overlap completely, their insert will be 50. Must be >, not >=')

    parser.add_argument('--store_everything', action='store_true', default=False, \
                        help="Store intermediate dictionaries in the pickle file; will result in significantly more RAM and disk usage")
    parser.add_argument('--skip_mm_profiling', action='store_true', default=False, \
                        help="Dont perform analysis on an mm level; saves RAM and time")

    # parser.add_argument("-g", "--genes", action="store", default=None, \
    #     help='Optional genes file')
    parser.add_argument('--debug', action='store_true', default=False, \
                        help="Produce some extra output helpful for debugging")

    # Parse
    if (len(args) == 0 or args[0] == '-h' or args[0] == '--help'):
        parser.print_help()
        sys.exit(0)
    else:
        return parser.parse_args(args)

if __name__ == '__main__':
    """ This is executed when run from the command line """

    pass
    # Controller().main(sys.argv[1:])
    # args = parse_arguments(sys.argv[1:])
    # main(args)

def _calc_counted_bases(basesCounted, maxMM):
    '''
    Return the number of bases counted at a particular mm level
    Returns the number of bases at this level and all levels below it
    '''
    counts = None
    for mm, count in [(mm, count) for mm, count in basesCounted.items() if mm <= maxMM]:
        if counts is None:
            counts = count
        else:
            counts = counts.add(count, fill_value=False)

    if counts is None:
        return 0

    else:
        return counts.astype('bool').sum()


import csv
import sys
import os

vendorList = []
#search = input("Enter vendor: ")
search = 'Cisco'
with open('vendors.csv', mode='r',encoding='utf-8') as vendorCSV:
    reader = csv.DictReader(vendorCSV)
    for line in reader:
        vendorList.append(line)
def vendorSearch(searchText):
    for vendor in vendorList:
        if searchText in vendor['Partner']:
            print ('Partner Account Manager: ' + vendor['Partner Account Manager'])
print (vendorSearch(search))


from django.shortcuts import render, get_object_or_404

# Create your views here.

from django.http import HttpResponse

from .models import Department, DepartmentSubPage

def departmentdetail(request):
  department = Department.objects.filter(url=request.path).first()
  departmentsubpage = DepartmentSubPage.objects.filter(url=request.path).first()
  if department:
    page = department
  elif departmentsubpage:
    page = departmentsubpage
  pageopts = page._meta
  department_subpages = DepartmentSubPage.objects.filter(parent__department__url=request.path).filter(deleted=0).filter(published=1)
  department_children = Department.objects.filter(parent__url=request.path)
  return render(request, 'departments/departmentdetail.html', {'page': page,'pageopts': pageopts,  'department_subpages': department_subpages, 'department_children': department_children})


def reg_plat():
    reg_nr = input("Skriv Reg. Nr: ")
    print(f'Bokstäver: {reg_nr[0:3]}\nSiffror: {reg_nr[3:6]}')

def numbers():
    numbers = input("Ange tal med komma imellan: ")
    string_nr = numbers.split(',')
    num = []
    for elem in string_nr:
        num.append(int(elem))
    print(f'Första talet: {numbers[0]}\nSista Talet: {numbers[-1]}\nSumma av Talen: {sum(num)}')
    print(f'Talen baklänges: {numbers[::-1]}')

def main():
    inp = input("Vill du köra 1. Reg_Plåt eller 2. Numbers: ")
    if inp == "1":
        reg_plat()
    if inp == "2":
        numbers()

if __name__ == '__main__':
    main()


from googleapiclient.discovery import build
import time
import pandas as pd
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class AutoCrawler:
    """If you enter YouTube keywords, the comment crawled data is saved as a csv file and provided._summary_
    """

    def __init__(self, DEVELOPER_KEY: str, CRHOM_PATH: str) -> None:
        """Enter the developer key of youtube data api v3 and the path of chorme.exe.

        :param DEVELOPER_KEY: Developer key of youtube data api v3
        :type DEVELOPER_KEY: str
        :param CRHOM_PATH: Path of chorme.exe.
        :type CRHOM_PATH: str
        """
        self._DEV_KEY = DEVELOPER_KEY
        self._CHROME = CRHOM_PATH
        self.YOUTUBE_API_SERVICE_NAME = "youtube"
        self.YOUTUBE_API_VERSION = "v3"

    def _get_links(self, search_keyword: str, max_link_len: int) -> list:
        """Retrieve video links by searching for a keyword on YouTube and limiting the number of links to max_link_len.

        :param search_keyword: The YouTube search keyword
        :type search_keyword: str
        :param max_link_len: Maximum returned link count
        :type max_link_len: int
        :return: List of searched video links
        :rtype: list
        """
        youtube = build(
            self.YOUTUBE_API_SERVICE_NAME,
            self.YOUTUBE_API_VERSION,
            developerKey=self._DEV_KEY,
        )
        search_response = (
            youtube.search()
            .list(
                q=search_keyword,
                type="video",
                part="id,snippet",
                maxResults=max_link_len,
            )
            .execute()
        )

        links = []
        for search_result in search_response.get("items", []):
            if search_result["id"]["kind"] == "youtube#video":
                links.append(
                    "https://www.youtube.com/watch?v={}".format(
                        search_result["id"]["videoId"]
                    )
                )

        return links

    def __get_comment(self, link: str, max_comment_pg_len: int) -> list:
        """This classmethod is used in the get_comment_df method.

        :param link: Link of YouTube video
        :type link: str
        :param max_comment_pg_len: Maximum number of comment pages to search
        :type max_comment_pg_len: int
        :return: List of comments in the video link
        :rtype: list
        """
        data = []
        with Chrome(executable_path=self._CHROME) as driver:
            wait = WebDriverWait(driver, 3)
            driver.get(link)
            for item in range(max_comment_pg_len):
                wait.until(
                    EC.visibility_of_element_located((By.TAG_NAME, "body"))
                ).send_keys(Keys.END)
                time.sleep(7)
            for comment in wait.until(
                EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#content"))
            ):
                data.append(comment.text)
        data = [x for x in data if x != ""]

        return data

    def _get_comment_df(self, links: list, max_comment_pg_len: int) -> pd.DataFrame:
        """This method crawls comments existing in multiple video links and creates a dataframe consisting of the 'comments' column containing the comments and the 'link' column containing the video link source of those comments.

        :param links: Links of YouTube video
        :type links: list
        :param max_comment_pg_len: Maximum number of comment pages to search
        :type max_comment_pg_len: int
        :return: a dataframe consisting of the 'comments' column containing the comments and the 'link' column containing the video link source of those comments.
        :rtype: pd.DataFrame
        """
        for i in range(len(links)):
            data = self.__get_comment(links[i], max_comment_pg_len)
            if i == 0:
                df = pd.DataFrame(data, columns=["comments"])
                df["link"] = links[0]
            else:
                temp_df = pd.DataFrame(data, columns=["comments"])
                temp_df["link"] = links[i]
                df = pd.concat([df, temp_df])
        df.dropna(how="any", inplace=True)

        return df

    def run(
        self,
        search_keyword: str,
        max_link_len: int,
        max_comment_pg_len: int,
        save_path: str,
    ) -> pd.DataFrame:
        """If you enter YouTube keywords, the comment crawled data is saved as a csv file and provided.

        :param search_keyword: The YouTube search keyword
        :type search_keyword: str
        :param max_link_len: Maximum returned link count
        :type max_link_len: int
        :param max_comment_pg_len: Maximum number of comment pages to search
        :type max_comment_pg_len: int
        :param save_path: Enter the full path to the csv file.
        :type save_path: str
        :return: Returns a pd.DataFrame object saved as a csv file.
        :rtype: pd.DataFrame
        """
        links = self._get_links(search_keyword, max_link_len)
        df = self._get_comment_df(links, max_comment_pg_len)
        if save_path is not None:
            try:
                df.to_csv(save_path, encoding="utf-8-sig", index=False)
                print(f"Save successed, Result dataframe save at {save_path}")
            except Exception as e:
                print(f"No folder exist: {save_path}, please enter right path. e.g. result/crawl.csv")

        return df


from buildcontrol.simple_ci.Master import Master
from buildcontrol.simple_ci.Slave import Slave
import sys
from core.helpers.GlobalMApp import mApp

def make_slave():
	# we have to copy some code here to instantiate the correct Simple CI instance (Master/Slave)
	for o in sys.argv[1:]:
		if o in ( "-s", "--slave" ):
			return True

	return False

if __name__ == "__main__":
	if make_slave():
		ci = Slave()
	else:
		ci = Master()

	# apply instance name if supplied
	if mApp().getParameters().getInstanceName():
		ci.setName( mApp().getParameters().getInstanceName() )

	ci.build()


# -*- coding: utf-8 -*-
"""
Created on Sun Jan 21 10:50:40 2018

@author: dell
"""
#导入库
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import tree
import graphviz
import seaborn as sns
from matplotlib.font_manager import FontProperties
myfont=FontProperties(fname=r'C:\Windows\Fonts\simhei.ttf',size=14)
sns.set(font=myfont.get_name())
#读取数据，测试集，训练集分开
train=pd.read_table('C:/Users/dell/Desktop/dzqh/data/contest_basic_train.tsv')
test=pd.read_table('C:/Users/dell/Desktop/dzqh/data/contest_basic_test.tsv')

#因为Y是目标变量值，所以Y单独一类
train_x=train.drop(['Y'],axis=1)
train_target=train['Y']
#先画几个图
"""train['LOAN_DATE'].value_counts().plot(kind="bar")
plt.title("放款时间")
plt.show()"""
#把本地的划归成1，非本地的划归成0
train.loc[(train['IS_LOCAL']=='本地籍'),'IS_LOCAL']=1
train.loc[(train['IS_LOCAL']=='非本地籍'),'IS_LOCAL']=0
train['IS_LOCAL']=train['IS_LOCAL'].astype(np.float64)
#把教育水平同样的区分，虽然有缺失值却在10%以内，可以把这一项当做零
#也可以说是单独看成一个分类
#print(train['EDU_LEVEL'].unique())#显示教育的种类
#教育水平跨分那么大可以考虑用one-hot matrix来预处理数据
train['EDU_LEVEL'].fillna(0,inplace=True)
train.loc[(train['EDU_LEVEL']=='其他'),'EDU_LEVEL']=0
train.loc[(train['EDU_LEVEL']=='初中'),'EDU_LEVEL']=1
train.loc[(train['EDU_LEVEL']=='专科及以下'),'EDU_LEVEL']=2
train.loc[(train['EDU_LEVEL']=='专科'),'EDU_LEVEL']=3
train.loc[(train['EDU_LEVEL']=='高中'),'EDU_LEVEL']=4
train.loc[(train['EDU_LEVEL']=='本科'),'EDU_LEVEL']=5
train.loc[(train['EDU_LEVEL']=='硕士研究生'),'EDU_LEVEL']=6
train.loc[(train['EDU_LEVEL']=='博士研究生'),'EDU_LEVEL']=7
train.loc[(train['EDU_LEVEL']=='硕士及以上'),'EDU_LEVEL']=6.5
train['EDU_LEVEL']=train['EDU_LEVEL'].astype(np.float64)

##关于婚姻状态的处理，把各种状态转化为数值型
train.loc[(train['MARRY_STATUS']=='已婚'),'MARRY_STATUS']=1
train.loc[(train['MARRY_STATUS']=='未婚'),'MARRY_STATUS']=2
train.loc[(train['MARRY_STATUS']=='离婚'),'MARRY_STATUS']=3
train.loc[(train['MARRY_STATUS']=='离异'),'MARRY_STATUS']=4
train.loc[(train['MARRY_STATUS']=='其他'),'MARRY_STATUS']=5
train.loc[(train['MARRY_STATUS']=='丧偶'),'MARRY_STATUS']=6
train['MARRY_STATUS']=train['MARRY_STATUS'].astype(np.float64)

train['WORK_PROVINCE']=train.fillna(method='bfill')
train['HAS_FUND']=train['HAS_FUND'].fillna(1.0)
#train['LOAN_DATE']=pd.to_datetime(train['LOAN_DATE'])

##要训练的数据
train_xx=train.drop(['Y','REPORT_ID','ID_CARD','LOAN_DATE','AGENT','SALARY'],axis=1)
#print(train_xx.info())

##采用决策树算法
clf=tree.DecisionTreeClassifier()
clf=clf.fit(train_xx,train_target)
print(clf.score(train_xx,train_target))


#!/usr/bin/env python3

import argparse

def main():
    parser = argparse.ArgumentParser(description='Small script to add an origin to out csv')
    parser.add_argument('--in_file', type=str, help='The input path of csv file to clean')
    parser.add_argument('--out_file', type=str, help='The output path of the script')
    parser.add_argument('--origin', type=str, help='Original repo where the file came from')
    args = parser.parse_args()
    if args.in_file == None:
        print('Please provide a input file path')
    if args.out_file == None:
        print('Please provide a output file path')
    if args.origin == None:
        print('Please provide an origin')
    add_origin(args.in_file, args.out_file, args.origin)

def add_origin(input_file, output_file, origin):
    with open(input_file, 'r') as input:
        with open(output_file, 'w') as output:
            output.writelines(input.readline()[:-1] + ',Origin\n')
            for line in input:
                output.writelines(line[:-1] + ',' + origin + '\n')

if __name__ == '__main__':
    main()


#jupyter magic
#wrote code below to file.py
# see new file!

from mrjob.job import MRJob

class MRWordFrequencyCount(MRJob):

    def mapper(self, _, line): # FOCUS ON
        #underscore means we will not be using this arg at all
        #go line by line in text
        yield "chars", len(line)
        yield "words", len(line.split()) #take a line and split by whitespace
        yield "lines", 1
        #yield creates a 'generator', we're making generators to not store stuff in memory

    def reducer(self, key, values ): #FOCUS ON
        yield key, sum(values)
                #values is a list of number of words

if __name__ == '__main__':
    MRWordFrequencyCount.run()


# import pygame module
import pygame

# Start pygame
pygame.init()

width = 800
height = 600

screen = pygame.display.set_mode((width, height))

cat = pygame.image.load('cat.png')
cat = pygame.transform.scale(cat, (120, 140))

# tracks whether the game is currently running
active = True

# This loop will run as long as active is True
while active:
    # get mouse position
    mouse_loc = pygame.mouse.get_pos()

    # Loop through the event queue. If we reach a QUIT event, set active to False
    # Once active is no longer True, the loop will no longer run
    for event in pygame.event.get():
        # Add a line of code here to print the event
        # print(event)

        if event.type == pygame.MOUSEBUTTONDOWN:
            print("clicked!")
            if event.button == 1:
                print("and it's the left button")

        if event.type == pygame.QUIT:
            active = False

    # display mouse location in terminal
    print(mouse_loc)
    # display cat
    screen.blit(cat, mouse_loc)

    # update the display
    pygame.display.flip()


import os
import sys

import discord
from discord import Activity, ActivityType
from discord.ext.commands import Bot
from dotenv import load_dotenv

load_dotenv()

if not os.path.isfile("config.json"):
    sys.exit("'config.json' not found! Please add it and try again.")
else:
    pass

client = Bot(command_prefix=">",
             help_command=None,
             case_insensitive=True,
             max_messages=100,
             activity=Activity(type=ActivityType.watching,
                               name=f"over Cartel."),
             intents=discord.Intents.all())

@client.event
async def on_ready():
    print("Main system is operational.")

for filename in os.listdir('./cogs'):
    if filename.endswith('.py'):
        client.load_extension(f'cogs.{filename[:-3]}')

TOKEN = os.getenv("DISCORD_TOKEN")
client.run(TOKEN)


##Lab 4: Loops and PI

##Prob 1 - DONE

def sum_nums(L):
  s = 0
  for num in L:
    s += num

  return s

def count_evens(L):
    count = 0
    for i in range(0, len(L)):
        if (L[i] % 2 == 0):
            count += 1
    return count

print(count_evens([1,2,3,4,5,6,7,8]))

##Prob 2 - DONE
#lis = L

def list_to_str(lis):
    #SHOULD DO: return(str(lis))
    if len(lis) == 0:
        return "'[]'"
    else:
        print("[", end="")
        for i in range(0, len(lis)-1):
            print(lis[i], " ", sep=",", end='')
        print(lis[len(lis)-1],"]", sep="")

list_to_str([1,2,3,4])

##Prob 3 - DONE
#lists_are_the_same(list1, list2), returns True iff list1 and list2 contain the same elements in the same order.

def lists_are_the_same(list1, list2):
    for i in range(0, len(list1)):
        if list1[i] != list2[i]:
            print("list item not the same")
        else:
            print("same")
lists_are_the_same([1,2,3,4],[1,2,4,3])

##Prob 4 - DONE
#simpify_fraction(n, m) which prints the simplified version of the fraction n/m
#ie. simpify_fraction(2, 6), print (1/2)
#Hint: use a similar technique to the one we used when determining whether a number is prime. That is, try dividing both the numerator and the denominator by every possible divisor in turn. For example, if you are simplifying 16/12, you can try dividing both by 16, 15, 14, ...., 1.

def simpify_fraction(n, m):
    if m % n == 0:
        m = int(m/n)
        n = int(n/n) #specify into so it doesn't return float
    print(str(n), "/", str(m), sep="")

simpify_fraction(4,8)

##Prob 5 - DONE
import math
pi = float(math.pi)
piCalc = pi * 10**15

sum = 0
terms = 100000
for t in range(terms+1):
    res = ((-1)**t)/(2*t + 1)
    sum += res
Lapprox = sum*4
LapproxCalc = Lapprox * 10**15

def find_same_sig_fig(x, y):
    sigfig = 0
    for i in range(15):
        a = x//10**(15-i)
        b = y//10**(15-i)
        if a == b:
            sigfig += 1
    return sigfig

print(find_same_sig_fig(LapproxCalc,piCalc))

##Prob 6 - DONE
#euclids algorithm: Given two numbers not prime to one another, to find their greatest common factor
#If A = 0 then GCD(A,B)=B, since the GCD(0,B)=B, and we can stop.
#If B = 0 then GCD(A,B)=A, since the GCD(A,0)=A, and we can stop.
#Write A in quotient remainder form (A = B⋅Q + R)
#Find GCD(B,R) using the Euclidean Algorithm since GCD(A,B) = GCD(B,R)

def euclids_alg(a,b):
    while a != 0:
        r = b % a
        b = a
        a = r
    print(b)

euclids_alg(1872,8493)


import numpy as np
import random

A = 7.9
B = 6.06
gamma = 85.8
radian = gamma / 180 * 3.14159

T1 = 400
T2 = 500
T3 = 600
Atoms = 100
eV_2_kcal_mol = 23.061
kB = 8.617E-5 * eV_2_kcal_mol

P_2_Q = np.array([-4, 2.5])

Q_in = np.array([[True,  True,  0,  1,  1.71,  0.0811,  0],
				 [True,  True,  0, -1,  1.71,  0.0811,  0],
				 [True,  True, -1,  1,  1.29,  0.0375,  0],
				 [True,  True,  1, -1,  1.29,  0.0375,  0],
				 [True,  True, -1,  0,  1.30,  0.00144,  0],
				 [True,  True,  1,  0,  1.30,  0.00144,  0],
				 [True,  False,  0, -1,  3.31,  0.104,  0],
				 [True,  False,  0,  1,  11.9,  0.581,  0],
				 [False,  False, -1,  0,  4.11E-2, -2.16E-4,  3.78E-7],
				 [False,  False, -1,  1,  2.67E-1, -1.14E-3,  1.31E-6],
				 [False,  False,  0,  0, -5.71E-2,  1.16E-3, -1.62E-6],
				 [False,  False, -1,  2,  1.71E-3, -1.16E-5,  1.98E-8]])

P_in = np.array([[True, False,  0,  1,  1.59,  0.103,  0],
				 [True, False,  0, -1,  1.59,  0.103,  0],
				 [True, False, -1,  1,  2.42,  0.0257, 0],
				 [True, False,  1, -1,  2.42,  0.0257, 0],
				 [True, False, -1,  0,  2.21,  0.0145, 0],
				 [True, False,  1,  0,  2.21,  0.0145, 0],
				 [True, True,  0, -1,  11.9,  0.581,  0],
				 [True, True,  0,  1,  3.31,  0.104,  0],
				 [False, True,  1, -2,  1.71E-3, -1.16E-5,  1.98E-8],
				 [False, True,  0,  0, -5.71E-2,  1.16E-3, -1.62E-6],
				 [False, True,  1, -1,  2.67E-1, -1.14E-3,  1.31E-6],
 				 [False, True,  1,  0,  4.11E-2, -2.16E-4,  3.78E-7]])

def Arr(E, A, T = T1, k = kB):
	return  A * np.exp(-E / (k * T))

def Poly(A, B, C, T = T1):
	return A + B * T + C * (T ** 2)

def iso_D(mean_r_squ, dt, n):
	return mean_r_squ / (2 * dn * t)

def dt(u, Rn):
	return -np.log([u]) / Rn

r_q = np.zeros((1, 12))
r_p = np.zeros((1, 12))

# Calculate rate of 12 steps for P and Q
for i in range(12):
	if Q_in[i][0]:
		r_q[0][i] = Arr(Q_in[i][4], Q_in[i][5])
	else:
		r_q[0][i] = Poly(Q_in[i][4], Q_in[i][5], Q_in[i][6])

for i in range(12):
	if P_in[i][0]:
		r_p[0][i] = Arr(P_in[i][4], P_in[i][5])
	else:
		r_p[0][i] = Poly(P_in[i][4], P_in[i][5], P_in[i][6])

# Calculate total rate for 1 atom
for i in range(1, 12):
	r_p[0][i] = r_p[0][i - 1] + r_p[0][i]
	r_q[0][i] = r_q[0][i - 1] + r_q[0][i]

q_minus_p = r_q[0][-1] - r_p[0][-1]

R_n = np.zeros(Atoms * 12 + 1)

for i in range(Atoms):
	for j in range(12):
		R_n[i * 12 + j + 1] = r_q[0][-1] * i + r_q[0][j]

# Multiply equal to 50,000,000
sampling_times = 10000
sampling_size = 5000

# Assume start from P point
At_Q = True
step_location = np.zeros((Atoms, 2))
delta_t = np.zeros(sampling_times)
PQ_check = np.ones((Atoms, 1), dtype=bool)

# x and y displacement
x = np.zeros((Atoms, 1))
y = np.zeros((Atoms, 1))

datax = np.zeros((sampling_times, Atoms))
datay = np.zeros((sampling_times, Atoms))

for i in range(sampling_times):
	for j in range(sampling_size):

		u_r = random.random()
		u_t = random.random()

		R_i = u_r * R_n[-1]
		R_temp = np.sort(np.hstack([R_n, [R_i]]))
		n_th = R_temp.tolist().index(R_i) - 1

		atom_num = int(n_th / 12)  # 0 - 99

		step_num = n_th % 12      # 0 - 11

		At_Q = np.copy(PQ_check[atom_num])

		if At_Q:
			Pts_in = Q_in
		else:
			Pts_in = P_in

		PQ_check[atom_num] = np.copy(Pts_in[step_num][1])
		step_location[atom_num] += np.array([Pts_in[step_num][2], Pts_in[step_num][3]])
		delta_t[i] += dt(u_t, R_n[-1])
		# Calculate new Rn

		# ########### Method 3 ###########
		if At_Q:
			if not PQ_check[atom_num]:
				for m in range(12):
					R_n[atom_num * 12 + m + 1] = np.copy(R_n[atom_num * 12] + r_q[0][m])
				for n in range((atom_num + 1) * 12 + 1, Atoms * 12 + 1):
					R_n[n] -= q_minus_p
		else:
			if PQ_check[atom_num]:
				for m in range(12):
					R_n[atom_num * 12 + m + 1] = np.copy(R_n[atom_num * 12] + r_p[0][m])
				for n in range((atom_num + 1) * 12 + 1, Atoms * 12 + 1):
					R_n[n] += q_minus_p
		################################

	for j in range(Atoms):
		x[j], y[j] = step_location[j][0] * A, step_location[j][1] * B
		if not PQ_check[j]:
			x[j] += P_2_Q[0]
			y[j] += P_2_Q[1]

	for j in range(Atoms):
		datax[i][j] = np.copy(x[j])
		datay[i][j] = np.copy(y[j])

np.savetxt('solux.csv', datax, delimiter=",")
np.savetxt('soluy.csv', datay, delimiter=",")
np.savetxt('dt.csv', delta_t, delimiter=",")


from itertools import permutations
def solution(k, dungeons):
    answer = -1
    nums = [k for k in range(len(dungeons))]
    perms = list(permutations(nums,len(dungeons)))
    result = [0 for i in range(len(perms))]
    for n,perm in enumerate(perms):
        temp = k
        for idx in perm:
            if temp >= dungeons[idx][0]:
                temp -= dungeons[idx][1]
                result[n] += 1
            else:
                break
    answer = max(result)
    return answer

k = 80
dungeons = [[80,20],[50,40],[30,10]]
print(solution(k, dungeons))


from tkinter import *
from quiz_brain import QuizBrain
import time
THEME_COLOR = "#375362"
FONT = ("Arial",20,"italic")

class QuizUI():

    def __init__(self,quiz_brain:QuizBrain):
        self.quiz_brain = quiz_brain

        self.window = Tk()
        self.window.config(padx=50,pady=50,bg=THEME_COLOR)
        self.window.title('Quizzlar')
        self.score = Label(text="Score: 0",bg=THEME_COLOR,fg='white')
        self.score.grid(row=0,column=1)

        self.canvas = Canvas(width=300,height=250,highlightthickness=0)
        self.Q_text = self.canvas.create_text(
            150,125,
            width=280,
            text="QUESTIONS",
            fill=THEME_COLOR,
            font=FONT)
        self.canvas.grid(row=1,column=0,columnspan=2,pady=50)

        #Buttons
        self.right_image = PhotoImage(file='images/true.png')
        self.right_btn = Button(image=self.right_image,highlightthickness=0,command=self.check_right_btn)
        self.right_btn.grid(row=2,column=0)

        self.left_image = PhotoImage(file='images/false.png')
        self.wrong_btn = Button(image=self.left_image,highlightthickness=0,command=self.check_wrong_btn)
        self.wrong_btn.grid(row=2,column=1)

        self.next_Q()
        self.window.mainloop()

    def next_Q(self):
        self.canvas.config(bg='white')
        if self.quiz_brain.still_has_questions():

            q_text = self.quiz_brain.next_question()
            self.canvas.itemconfig(self.Q_text,text=q_text)
        else:
            self.canvas.itemconfig(self.Q_text,text="You have reached end of quiz")
            self.right_btn.config(state='disabled')
            self.wrong_btn.config(state='disabled')

    def check_right_btn(self):
        answer = self.quiz_brain.check_answer("true")
        self.feeback(answer)

    def check_wrong_btn(self):
        answer = self.quiz_brain.check_answer("false")
        self.feeback(answer)

    def feeback(self,param: bool):
        if param:
            self.canvas.config(bg='green')
            self.score['text'] = f"Score: {self.quiz_brain.score}"
        else:
            self.canvas.config(bg='red')
        self.window.after(1000,self.next_Q)


l,r=[int(c) for c in input().split()]
c=0
for num in range(l,r+1):
    f = 0
    for i in range(2, num):
        if num % i == 0:
            f = 1
            break
    if f == 0:
        c+=1
print(c)


l = []
c = input().strip()
while c != 'end':
	if c == 'list':
		print(l)
	elif c == 'shelf':
		if len(l)>0:
			x=l.pop(len(l)-1)
			print(x)
		else:
			print("no book")
	elif c == 'top':
		if len(l)>0:
			print(l[len(l)-1])
		else:
			print("no book")
	else:
		l.append(c[7:])
		print(len(l))
	c = input().strip()


import os
import pickle

import imageio as imageio
import matplotlib.pyplot as plt
import numpy as np

from util.downloader import maybe_download_and_extract
from util.image_utils import pad_images

def write_image_matrix(array, name, dst_folder=None):
    if dst_folder is None:
        dst_folder = './image_out/'
    else:
        dst_folder = os.path.join(os.getcwd(), dst_folder)
    if not os.path.exists(dst_folder):
        os.makedirs(dst_folder)
    imageio.imwrite(os.path.join(dst_folder, name + '.png'), array)

def unpickle(file):
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

def do_pickle(file, obj):
    with open(file, 'wb') as fo:
        pickle.dump(obj, fo, protocol=pickle.HIGHEST_PROTOCOL)
    return dict

def _maybe_download_cifar10_data():
    path = './data/cifar10'

    if not os.path.exists(path):
        os.makedirs(path)

    maybe_download_and_extract(path)

def get_cifar10_dict():
    data = {}
    path = os.path.join(os.getcwd(), './data/cifar10/cifar-10-batches-py')

    _maybe_download_cifar10_data()

    file_names = [os.path.join(path, f) for f in os.listdir(path)
                  if os.path.isfile(os.path.join(path, f)) and "data_batch" in f]

    for file_path in file_names:
        images, labels = cifar10_batch_to_matrix(unpickle(file_path))
        for i in range(len(images)):
            if str(labels[i]) in data:
                data[str(labels[i])].append(images[i])
            else:
                data[str(labels[i])] = [images[i]]

    for k, val in data.items():
        data[k] = np.array(val)
    return data

def get_cifar10_train_and_valid(standardize=True):
    data = get_cifar10_dict()
    m_data = None
    labels = None

    valid_data = None
    valid_labels = None

    for k, val in data.items():
        if standardize:
            val = convert_uint_to_tanh_range(np.array(val))
        mask = dict(zip(np.random.choice(5000, 500, replace=False), np.ones(500)))
        label_int = int(float(k))
        l = np.ones(4500) * label_int
        v_l = np.ones(500) * label_int

        valid = val[:500]
        train = val[500:]

        if m_data is None:
            valid_data = valid  # np.array([e for i, e in enumerate(val) if i in mask])
            valid_labels = v_l

            m_data = train  # np.array([e for i, e in enumerate(val) if i not in mask])
            labels = l

        else:
            valid_data = np.vstack((valid_data, valid))
            valid_labels = np.concatenate((valid_labels, v_l))

            m_data = np.vstack([m_data, train])
            labels = np.concatenate((labels, l))

    assert len(m_data) == 45000
    assert len(labels) == 45000
    assert len(valid_data) == 5000
    assert len(valid_labels) == 5000

    # i = 0
    # while i < 5000:
    #     print('image at i=%d is of class %d' % (i, valid_labels[i]))
    #     sample = valid_data[i]
    #     sample = (sample + 1) / 2
    #
    #     plt.imshow(sample)
    #
    #     plt.show()
    #     i += 500

    return m_data, labels.astype(np.int32), valid_data, valid_labels.astype(np.int32)

def cifar10_batch_to_matrix(data):
    m_data = np.swapaxes(np.swapaxes(data[b'data'].reshape([len(data[b'data']), 3, 32, 32]), 1, 3), 1, 2)
    m_labels = np.array(data[b'labels'], dtype=np.float32)
    return m_data, m_labels

def combine_image_arrays(data, img_dim):
    assert len(img_dim) == 2, "img_dim should be length 2 array"
    batch_size, x_dim, y_dim, channels = data.shape

    h, w = [round(e) for e in img_dim]
    # assert batch_size == h * w, "batch_size should equal img_dim[0] * img_dim[1]"

    matrix = np.array([])
    for i in range(h):
        row_array = [np.reshape(data[i * w + j], [x_dim, y_dim, channels]) for j in range(w)]
        row = np.concatenate(row_array, axis=1)
        if len(matrix) == 0:
            matrix = row
        else:
            matrix = np.vstack([matrix, row])
    return matrix.astype(np.uint8)

def convert_uint_to_tanh_range(images):
    return (images / 127.5) - 1

def convert_tanh_float_to_uint(images):
    return (images + 1) * 127.5

def get_cifar10_batch():
    data = []
    path = os.path.join(os.getcwd(), './data/cifar10/cifar-10-batches-py')
    _maybe_download_cifar10_data()
    file_names = [os.path.join(path, f) for f in os.listdir(path)
                  if os.path.isfile(os.path.join(path, f)) and "data_batch" in f]
    for file_path in file_names:
        data.append(unpickle(file_path))
    return data, unpickle(os.path.join(path, 'test_batch'))

def get_cifar10_all_data():
    path = os.path.join(os.getcwd(), './data/cifar10/cifar-10-batches-py')
    _maybe_download_cifar10_data()
    file_names = [os.path.join(path, f) for f in os.listdir(path)
                  if os.path.isfile(os.path.join(path, f)) and "data_batch" in f]
    m_data = None
    labels = None
    for name in file_names:
        data = unpickle(name)
        features, l = cifar10_batch_to_matrix(data)
        if m_data is None:
            m_data = features
            labels = l
        else:
            m_data = np.vstack((m_data, features))
            labels = np.concatenate((labels, l))
    return m_data, labels

def get_cifar10_test_data():
    path = os.path.join(os.getcwd(), './data/cifar10/cifar-10-batches-py')
    _maybe_download_cifar10_data()
    file_names = [os.path.join(path, f) for f in os.listdir(path)
                  if os.path.isfile(os.path.join(path, f)) and "test_batch" in f]
    m_data = None
    labels = None
    for name in file_names:
        data = unpickle(name)
        features, l = cifar10_batch_to_matrix(data)
        if m_data is None:
            m_data = features
            labels = l
        else:
            m_data = np.vstack((m_data, features))
            labels = np.concatenate((labels, l))
    return m_data, labels

def get_padded_cifar10_data(shuffle=True):
    path = os.path.join(os.getcwd(), './data/cifar10_padded')
    valid_file = os.path.join(path, 'valid_batch')

    if not os.path.exists(path) or len(os.listdir(path)) == 0:
        if not os.path.exists(path):
            os.makedirs(path)

        m_data, labels, valid_data, valid_labels = get_cifar10_train_and_valid()

        m_data = pad_images(m_data)

        if shuffle:
            # shuffle here
            i = np.arange(m_data.shape[0])
            np.random.shuffle(i)

            m_data = m_data[i]
            labels = labels[i]

            i = np.arange(valid_data.shape[0])
            np.random.shuffle(i)

            valid_data = valid_data[i]
            valid_labels = valid_labels[i]

        store_size = len(m_data) // 10

        for i in range(10):
            start = i * store_size
            end = (i + 1) * store_size
            if end > len(m_data):
                end = len(m_data)
            store_obj = {'data': m_data[start:end], 'labels': labels[start:end]}
            do_pickle(os.path.join(path, 'data_batch' + str(i)), store_obj)

        # store valid dictionary
        valid_obj = {'data': valid_data, 'labels': valid_labels}
        do_pickle(valid_file, valid_obj)

        return m_data, labels, valid_data, valid_labels

    file_names = [os.path.join(path, f) for f in os.listdir(path)
                  if os.path.isfile(os.path.join(path, f)) and "data_batch" in f]
    m_data = None
    labels = None
    for name in file_names:
        data = unpickle(name)
        features = data['data']
        l = data['labels']

        if m_data is None:
            m_data = features
            labels = l
        else:
            m_data = np.vstack((m_data, features))
            labels = np.concatenate((labels, l))

    obj = unpickle(valid_file)
    valid_data = obj['data']
    valid_labels = obj['labels']

    return m_data, labels, valid_data, valid_labels


import re
import pytest
from itertools import chain

pattern = r"(\(\?[^:=!<].*?\))|(\([^\?].*?\))"

test_cases = [
    ("back(boards|block)", ["(boards|block)"]),
    ("(?:non-capturing)(capturing group)", ["(capturing group)"]),
    ("(\w+)(?=\d)", ["(\w+)"]),
    ("(?P<year>\d{4})-(?P<month>\d{2})(?! [a-z])", ["(?P<year>\d{4})", "(?P<month>\d{2})"]),
    ("(?<=[A-Z])1234 teach(er|ing)", ["(er|ing)"]),
    ("(?<!\d)", [None]),
    ("(?P=name\w+)", ["(?P=name\w+)"])
]

@pytest.mark.parametrize("string, answer", test_cases)
def test_group_capturing(string, answer):
    """Test whether pattern correctly matches or does not match input."""
    nested_list = [[x for x in y if len(x) != 0] for y in re.findall(pattern, string)]
    flat_list = list(chain(*nested_list))
    s = zip(flat_list, answer)
    for el in s:
        print(el)
        assert el[0] == el[1]