import tensorflow.keras.backend as K
import tensorflow as tf

from tensorflow_addons.layers import InstanceNormalization
from tensorflow.keras.layers import Layer, Input, Conv2D, Activation, add, BatchNormalization, UpSampling2D, ZeroPadding2D, Conv2DTranspose, Flatten, MaxPooling2D, AveragePooling2D, InputSpec, LeakyReLU, Dense

from tensorflow.keras.initializers import RandomNormal
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.backend import mean
from tensorflow.keras.models import Model, model_from_json
from tensorflow.keras.utils import plot_model
from tensorflow.python.keras.engine.network import Network

import random
import datetime
import time
import json
import math
import csv
import sys
from os.path import join, exists
from os import mkdir, makedirs
import numpy as np

# sys.path.append('../')
#import load_data

from factories.optimizer_factory import Optimizer_Factory
from layers.reflection_padding_2d import ReflectionPadding2D
from metrics.loss_functions import lse, cycle_loss

def save_img(img, fname):
    if len(tf.shape(img)) > 3:
        img = img[tf.constant(0),:,:,:]
    img = (img + 1) / 2.0
    result_img = tf.image.encode_png(tf.image.convert_image_dtype(img, dtype=tf.uint8))
    tf.io.write_file(tf.constant(fname), result_img)

class ImagePool():
    def __init__(self, pool_size):
        self.pool_size = pool_size
        if self.pool_size > 0:
            self.num_imgs = 0
            self.images = []

    def query(self, images):
        if self.pool_size == 0:
            return images
        return_images = []
        for image in images:
            if len(image.shape) == 3:
                image = image[np.newaxis, :, :, :]

            if self.num_imgs < self.pool_size:  # fill up the image pool
                self.num_imgs = self.num_imgs + 1
                if len(self.images) == 0:
                    self.images = image
                else:
                    self.images = np.vstack((self.images, image))

                if len(return_images) == 0:
                    return_images = image
                else:
                    return_images = np.vstack((return_images, image))

            else:  # 50% chance that we replace an old synthetic image
                p = random.uniform(0, 1)
                if p > 0.5:
                    random_id = random.randint(0, self.pool_size - 1)
                    tmp = self.images[random_id, :, :, :]
                    tmp = tmp[np.newaxis, :, :, :]
                    self.images[random_id, :, :, :] = image[0, :, :, :]
                    if len(return_images) == 0:
                        return_images = tmp
                    else:
                        return_images = np.vstack((return_images, tmp))
                else:
                    if len(return_images) == 0:
                        return_images = image
                    else:
                        return_images = np.vstack((return_images, image))

        return return_images

class CycleGAN():
    """docstring for CycleGAN."""

    def __init__(self, cf):
        self.model = None
        self.cf = cf

        # from Keras documentation: Total number of steps (batches of samples) to yield from generator before declaring one epoch finished and starting the next epoch. It should typically be equal to the number of unique samples of your dataset divided by the batch size.
        if(cf.n_images_train is not None):
            self.steps_per_epoch = int(np.ceil(cf.n_images_train / float(cf.batch_size_train)))
        else:
            samplesTrainA = self.get_file_len(cf.trainA_file_path_full)
            samplesTrainB = self.get_file_len(cf.trainB_file_path_full)
            samplesTrain = max(samplesTrainA, samplesTrainB)
            self.steps_per_epoch = int(np.ceil(samplesTrain / float(cf.batch_size_train)))
            # zzh
            # self.steps_per_epoch = int(np.ceil(get_file_len(cf.train_file_path_full) / float(cf.batch_size_train)))
            # self.validation_steps = int(np.ceil(get_file_len(cf.valid_file_path_full) / float(cf.batch_size_valid)))
            # samplesTestA = self.get_file_len(cf.testA_file_path_full)
            # samplesTestB = self.get_file_len(cf.testB_file_path_full)
            # samplesTest = max(samplesTestA, samplesTestB)
            # self.test_steps = int(np.ceil(samplesTest / float(cf.batch_size_valid)))

    def get_file_len(self, file_path_full):
        return len(["" for line in open(file_path_full, "r")])

    def make_discriminators(self):
        # ======= Discriminator model ==========
        if self.cf.use_multiscale_discriminator:
            D_A = self.modelMultiScaleDiscriminator()
            D_B = self.modelMultiScaleDiscriminator()
            loss_weights_D = [0.5, 0.5] # 0.5 since we train on real and synthetic images
        else:
            D_A = self.modelDiscriminator()
            D_B = self.modelDiscriminator()
            loss_weights_D = [0.5]  # 0.5 since we train on real and synthetic images

        # Discriminator builds
        image_A = Input(shape=self.cf.input_shape)
        image_B = Input(shape=self.cf.input_shape)
        guess_A = D_A(image_A)
        guess_B = D_B(image_B)
        self.D_A = Model(inputs=image_A, outputs=guess_A, name='D_A_model')
        self.D_B = Model(inputs=image_B, outputs=guess_B, name='D_B_model')

        # self.D_A.summary()
        # self.D_B.summary()
        optimizer_D = Optimizer_Factory().make(self.cf.optimizer, self.cf.learning_rate_D)
        self.D_A.compile(optimizer=optimizer_D,
                         loss=lse,
                         loss_weights=loss_weights_D)
        self.D_B.compile(optimizer=optimizer_D,
                         loss=lse,
                         loss_weights=loss_weights_D)

        # Use Networks to avoid falsy keras error about weight descripancies
        self.D_A_static = Network(inputs=image_A, outputs=guess_A, name='D_A_static_model')
        self.D_B_static = Network(inputs=image_B, outputs=guess_B, name='D_B_static_model')

        # Do note update discriminator weights during generator training
        self.D_A_static.trainable = False
        self.D_B_static.trainable = False

    def make_generators(self):
        # Generators
        self.G_A2B = self.modelGenerator(name='G_A2B_model')
        self.G_B2A = self.modelGenerator(name='G_B2A_model')
        # self.G_A2B.summary()

        optimizer_G = Optimizer_Factory().make(self.cf.optimizer, self.cf.learning_rate_G)

        if self.cf.use_identity_learning:
            self.G_A2B.compile(optimizer=optimizer_G, loss='MAE')
            self.G_B2A.compile(optimizer=optimizer_G, loss='MAE')

        # Generator builds
        real_A = Input(shape=self.cf.input_shape, name='real_A')
        real_B = Input(shape=self.cf.input_shape, name='real_B')
        synthetic_B = self.G_A2B(real_A)
        synthetic_A = self.G_B2A(real_B)
        dA_guess_synthetic = self.D_A_static(synthetic_A)
        dB_guess_synthetic = self.D_B_static(synthetic_B)
        reconstructed_A = self.G_B2A(synthetic_B)
        reconstructed_B = self.G_A2B(synthetic_A)

        model_outputs = [reconstructed_A, reconstructed_B]
        compile_losses = [cycle_loss, cycle_loss,
                          lse, lse]
        compile_weights = [self.cf.lambda_1, self.cf.lambda_2,
                           self.cf.lambda_D, self.cf.lambda_D]

        if self.cf.use_multiscale_discriminator:
            for _ in range(2):
                compile_losses.append(lse)
                compile_weights.append(self.cf.lambda_D)  # * 1e-3)  # Lower weight to regularize the model
            for i in range(2):
                model_outputs.append(dA_guess_synthetic[i])
                model_outputs.append(dB_guess_synthetic[i])
        else:
            model_outputs.append(dA_guess_synthetic)
            model_outputs.append(dB_guess_synthetic)

        if self.cf.use_supervised_learning:
            model_outputs.append(synthetic_A)
            model_outputs.append(synthetic_B)
            compile_losses.append('MAE')
            compile_losses.append('MAE')
            compile_weights.append(self.cf.supervised_weight)
            compile_weights.append(self.cf.supervised_weight)

        self.model = Model(inputs=[real_A, real_B],
                             outputs=model_outputs,
                             name='G_model')

        self.model.compile(optimizer=optimizer_G,
                             loss=compile_losses,
                             loss_weights=compile_weights)
        # self.G_A2B.summary()

    def make(self):
        # Build model
        self.make_discriminators()
        print(' Build discriminators successfully...')
        # make generators
        self.make_generators()
        print(' Build generators successfully...')

        # Load pretrained weights
        # zzh
        if not self.cf.load_pretrained:
            print(' No loading pretrained weights...')
        elif not exists(join(self.cf.savepath, 'pretrained_weights')):
            print(' No pretrained weights exist...')
            return
        else:
            self.load_pretrained_weights(self.G_A2B)
            self.load_pretrained_weights(self.G_B2A)
            self.load_pretrained_weights(self.D_A)
            self.load_pretrained_weights(self.D_B)
            print(' Load pretrained weights successfully...')

        # if self.cf.resume_training:
        #     model.load_weights(cf.checkpoint_path)

        # Show model structure
        if self.cf.show_model:
            model.summary()
            plot_model(model, to_file=join(self.cf.savepath, 'model.png'))

        # Output the model
        print('   Model: ' + self.cf.model_name)

    # Learning rate #
    def get_lr_linear_decay_rate(self, max_nr_images):
        # Calculate decay rates
        updates_per_epoch_D = 2 * max_nr_images + self.cf.discriminator_iterations - 1
        updates_per_epoch_G = max_nr_images + self.cf.generator_iterations - 1
        if self.cf.use_identity_learning:
            updates_per_epoch_G *= (1 + 1 / self.cf.identity_mapping_modulus)
        denominator_D = (self.cf.n_epochs - self.cf.decay_epoch) * updates_per_epoch_D
        denominator_G = (self.cf.n_epochs - self.cf.decay_epoch) * updates_per_epoch_G
        decay_D = self.cf.learning_rate_D / denominator_D
        decay_G = self.cf.learning_rate_G / denominator_G

        return decay_D, decay_G

    def update_lr(self, model, decay):
        new_lr = K.get_value(model.optimizer.lr) - decay
        if new_lr < 0:
            new_lr = 0
        # print(K.get_value(model.optimizer.lr))
        K.set_value(model.optimizer.lr, new_lr)

    # Images #
    def save_tmp_images(self, real_image_A, real_image_B, synthetic_image_A, synthetic_image_B):
        try:
            reconstructed_image_A = self.G_B2A.predict(synthetic_image_B)
            reconstructed_image_B = self.G_A2B.predict(synthetic_image_A)

            real_images = np.vstack((real_image_A[0], real_image_B[0]))
            synthetic_images = np.vstack((synthetic_image_B[0], synthetic_image_A[0]))
            reconstructed_images = np.vstack((reconstructed_image_A[0], reconstructed_image_B[0]))

            # save path
            result_path = join(self.cf.savepath, 'tmp_images')
            if exists(result_path) == False:
                mkdir(result_path)

            # write images
            save_img(real_image_A[0,:,:,:], join(result_path, 'realA.png'))
            save_img(real_image_B[0,:,:,:], join(result_path, 'realB.png'))
            save_img(synthetic_image_A[0,:,:,:], join(result_path, 'syntheticA.png'))
            save_img(synthetic_image_B[0,:,:,:], join(result_path, 'syntheticB.png'))
            save_img(reconstructed_image_A[0,:,:,:], join(result_path, 'reconstructedA.png'))
            save_img(reconstructed_image_B[0,:,:,:], join(result_path, 'reconstructedB.png'))

        except: # Ignore if file is open
            pass

    def saveImages(self, epoch, real_image_A, real_image_B, num_saved_images=1):
        result_path = join(self.cf.savepath, 'images')
        # if not exists(join(result_path, 'A')):
        #     makedirs(join(result_path, 'A'))
        #     makedirs(join(result_path, 'B'))
        #     makedirs(join(result_path, 'Atest'))
        #     makedirs(join(result_path, 'Btest'))

        for i in range(num_saved_images):
            # if i == num_saved_images:
            #     real_image_A = self.A_test[0]
            #     real_image_B = self.B_test[0]
            #     real_image_A = np.expand_dims(real_image_A, axis=0)
            #     real_image_B = np.expand_dims(real_image_B, axis=0)
            #     testString = 'test'
            #
            # else:
            #     #real_image_A = self.A_train[rand_A_idx[i]]
            #     #real_image_B = self.B_train[rand_B_idx[i]]
            #     if len(real_image_A.shape) < 4:
            #         real_image_A = np.expand_dims(real_image_A, axis=0)
            #         real_image_B = np.expand_dims(real_image_B, axis=0)

            synthetic_image_B = self.G_A2B.predict(real_image_A)
            synthetic_image_A = self.G_B2A.predict(real_image_B)
            reconstructed_image_A = self.G_B2A.predict(synthetic_image_B)
            reconstructed_image_B = self.G_A2B.predict(synthetic_image_A)

            # write images
            print("SHAPE")
            print(tf.shape(real_image_A))
            save_img(real_image_A[i,:,:,:], join(result_path, str(epoch) + '_' + str(i) + '_realA.png'))
            save_img(real_image_B[i,:,:,:], join(result_path, str(epoch) + '_' + str(i) + '_realB.png'))
            save_img(synthetic_image_A[i,:,:,:], join(result_path, str(epoch) + '_' + str(i) + '_syntheticA.png'))
            save_img(synthetic_image_B[i,:,:,:], join(result_path, str(epoch) + '_' + str(i) + '_syntheticB.png'))
            save_img(reconstructed_image_A[i,:,:,:], join(result_path, str(epoch) + '_' + str(i) + '_reconstructedA.png'))
            save_img(reconstructed_image_B[i,:,:,:], join(result_path, str(epoch) + '_' + str(i) + '_reconstructedB.png'))

    # Training #
    def print_ETA(self, start_time, epoch, loop_index):
        passed_time = time.time() - start_time

        iterations_so_far = ((epoch - 1) * self.steps_per_epoch + loop_index) / self.cf.batch_size_train
        iterations_total = self.cf.n_epochs * self.steps_per_epoch / self.cf.batch_size_train
        iterations_left = iterations_total - iterations_so_far
        eta = round(passed_time / (iterations_so_far + 1e-5) * iterations_left)

        passed_time_string = str(datetime.timedelta(seconds=round(passed_time)))
        eta_string = str(datetime.timedelta(seconds=eta))
        print('Time passed', passed_time_string, ': ETA in', eta_string)

    def _run_training_iteration(self, loop_index, epoch, real_images_A, real_images_B, ones, zeros, synthetic_pool_A, synthetic_pool_B):
        # ======= Discriminator training ==========
            # Generate batch of synthetic images
        synthetic_images_B = self.G_A2B.predict(real_images_A)
        synthetic_images_A = self.G_B2A.predict(real_images_B)
        synthetic_images_A = synthetic_pool_A.query(synthetic_images_A)
        synthetic_images_B = synthetic_pool_B.query(synthetic_images_B)

        for _ in range(self.cf.discriminator_iterations):
            DA_loss_real = self.D_A.train_on_batch(x=real_images_A, y=ones)
            DB_loss_real = self.D_B.train_on_batch(x=real_images_B, y=ones)
            DA_loss_synthetic = self.D_A.train_on_batch(x=synthetic_images_A, y=zeros)
            DB_loss_synthetic = self.D_B.train_on_batch(x=synthetic_images_B, y=zeros)
            if self.cf.use_multiscale_discriminator:
                DA_loss = sum(DA_loss_real) + sum(DA_loss_synthetic)
                DB_loss = sum(DB_loss_real) + sum(DB_loss_synthetic)
                print('DA_losses: ', np.add(DA_loss_real, DA_loss_synthetic))
                print('DB_losses: ', np.add(DB_loss_real, DB_loss_synthetic))
            else:
                DA_loss = DA_loss_real + DA_loss_synthetic
                DB_loss = DB_loss_real + DB_loss_synthetic
            D_loss = DA_loss + DB_loss

            if self.cf.discriminator_iterations > 1:
                print('D_loss:', D_loss)
                sys.stdout.flush()

        '''
        # zzh
        directory = join(self.cf.savepath, 'saved_models')
        if not exists(directory):
            makedirs(directory)
        temp_D_A_PATH = join(directory,'temp_D_A')
        temp_D_B_PATH = join(directory, 'temp_D_B')
        self.D_A.save_weights(temp_D_A_PATH)
        self.D_A_static.load_weights(temp_D_A_PATH)
        self.D_B.save_weights(temp_D_B_PATH)
        self.D_B_static.load_weights(temp_D_B_PATH)
        '''
        # ======= Generator training ==========
        target_data = [real_images_A, real_images_B]  # Compare reconstructed images to real images
        if self.cf.use_multiscale_discriminator:
            for i in range(2):
                target_data.append(ones[i])
                target_data.append(ones[i])
        else:
            target_data.append(ones)
            target_data.append(ones)

        if self.cf.use_supervised_learning:
            target_data.append(real_images_A)
            target_data.append(real_images_B)

        for _ in range(self.cf.generator_iterations):
            G_loss = self.model.train_on_batch(x=[real_images_A, real_images_B], y=target_data)
            if self.cf.generator_iterations > 1:
                print('G_loss:', G_loss)
                sys.stdout.flush()

        gA_d_loss_synthetic = G_loss[1]
        gB_d_loss_synthetic = G_loss[2]
        reconstruction_loss_A = G_loss[3]
        reconstruction_loss_B = G_loss[4]

        # Identity training
        if self.cf.use_identity_learning and loop_index % self.cf.identity_mapping_modulus == 0:
            G_A2B_identity_loss = self.G_A2B.train_on_batch(
                x=real_images_B, y=real_images_B)
            G_B2A_identity_loss = self.G_B2A.train_on_batch(
                x=real_images_A, y=real_images_A)
            print('G_A2B_identity_loss:', G_A2B_identity_loss)
            print('G_B2A_identity_loss:', G_B2A_identity_loss)

        # Update learning rates
        if self.cf.use_linear_decay and epoch > self.cf.decay_epoch:
            self.update_lr(self.D_A, self.decay_D)
            self.update_lr(self.D_B, self.decay_D)
            self.update_lr(self.model, self.decay_G)

        # Store training data
        self.training_history['DA_losses'].append(DA_loss)
        self.training_history['DB_losses'].append(DB_loss)
        self.training_history['gA_d_losses_synthetic'].append(gA_d_loss_synthetic)
        self.training_history['gB_d_losses_synthetic'].append(gB_d_loss_synthetic)
        self.training_history['gA_losses_reconstructed'].append(reconstruction_loss_A)
        self.training_history['gB_losses_reconstructed'].append(reconstruction_loss_B)

        GA_loss = gA_d_loss_synthetic + reconstruction_loss_A
        GB_loss = gB_d_loss_synthetic + reconstruction_loss_B
        reconstruction_loss = reconstruction_loss_A + reconstruction_loss_B

        self.training_history['D_losses'].append(D_loss)
        self.training_history['G_losses'].append(G_loss)
        self.training_history['reconstruction_losses'].append(reconstruction_loss)

        # GA_losses.append(GA_loss)
        # GB_losses.append(GB_loss)

        print('\n')
        print('Epoch----------------', epoch, '/', self.cf.n_epochs)
        print('Loop index----------------', loop_index + 1, '/', self.steps_per_epoch)
        print('D_loss: ', D_loss)
        print('G_loss: ', G_loss[0])
        print('reconstruction_loss: ', reconstruction_loss)
        print('DA_loss:', DA_loss)
        print('DB_loss:', DB_loss)

        if loop_index % 20 == 0:
            # Save temporary images continously
            #self.save_tmp_images(real_images_A, real_images_B, synthetic_images_A, synthetic_images_B)
            self.print_ETA(self.start_time, epoch, loop_index)

    def train(self, train_gen, cb):
        if (not self.cf.train_model):
            return None

        print('\n > Training the model...')
        # ======================================================================
        # Begin training
        # ======================================================================
        training_history = {}

        DA_losses = []
        DB_losses = []
        gA_d_losses_synthetic = []
        gB_d_losses_synthetic = []
        gA_losses_reconstructed = []
        gB_losses_reconstructed = []

        GA_losses = []
        GB_losses = []
        reconstruction_losses = []
        D_losses = []
        G_losses = []

        self.training_history = {
            'DA_losses': DA_losses,
            'DB_losses': DB_losses,
            'gA_d_losses_synthetic': gA_d_losses_synthetic,
            'gB_d_losses_synthetic': gB_d_losses_synthetic,
            'gA_losses_reconstructed': gA_losses_reconstructed,
            'gB_losses_reconstructed': gB_losses_reconstructed,
            'D_losses': D_losses,
            'G_losses': G_losses,
            'reconstruction_losses': reconstruction_losses}

        # Image pools used to update the discriminators
        synthetic_pool_A = ImagePool(self.cf.synthetic_pool_size)
        synthetic_pool_B = ImagePool(self.cf.synthetic_pool_size)

        # self.saveImages('(init)')

        # Tweaks
        self.REAL_LABEL = 1.0  # Use e.g. 0.9 to avoid training the discriminators to zero loss

        # labels
        if self.cf.use_multiscale_discriminator:
            label_shape1 = (self.cf.batch_size_train,) + self.D_A.output_shape[0][1:]
            label_shape2 = (self.cf.batch_size_train,) + self.D_A.output_shape[1][1:]
            #label_shape4 = (self.cf.batch_size_train,) + self.D_A.output_shape[2][1:]
            ones1 = np.ones(shape=label_shape1) * self.REAL_LABEL
            ones2 = np.ones(shape=label_shape2) * self.REAL_LABEL
            #ones4 = np.ones(shape=label_shape4) * self.REAL_LABEL
            ones = [ones1, ones2]  # , ones4]
            zeros1 = ones1 * 0
            zeros2 = ones2 * 0
            #zeros4 = ones4 * 0
            zeros = [zeros1, zeros2]  # , zeros4]
        else:
            label_shape = (self.cf.batch_size_train,) + self.D_A.output_shape[1:]
            ones = np.ones(shape=label_shape) * self.REAL_LABEL
            zeros = ones * 0

        # Linear decay
        if self.cf.use_linear_decay:
            self.decay_D, self.decay_G = self.get_lr_linear_decay_rate(self.steps_per_epoch * self.cf.n_epochs)

        # Start stopwatch for ETAs
        self.start_time = time.time()

        for epoch in range(1, self.cf.n_epochs + 1):
            loop_index = 1
            for images in train_gen:
                real_images_A = images[0]
                real_images_B = images[1]

                # Run all training steps
                self._run_training_iteration(loop_index, epoch, real_images_A, real_images_B, ones, zeros, synthetic_pool_A, synthetic_pool_B)

                # Store models
                if loop_index % 20000 == 0:
                    self.save_model(self.D_A, loop_index)
                    self.save_model(self.D_B, loop_index)
                    self.save_model(self.G_A2B, loop_index)
                    self.save_model(self.G_B2A, loop_index)

                # Break if loop has ended
                if loop_index >= self.steps_per_epoch:
                    break

                loop_index += 1

            #================== within epoch loop end ==========================

            if epoch % self.cf.save_interval == 0:
                print('\n', '\n', '-------------------------Saving images for epoch', epoch, '-------------------------', '\n', '\n')
                self.saveImages(epoch, real_images_A, real_images_B, tf.shape(real_images_A)[0])

            if epoch % 5 == 0:
                # self.save_model(self.model)
                self.save_model(self.D_A, epoch)
                self.save_model(self.D_B, epoch)
                self.save_model(self.G_A2B, epoch)
                self.save_model(self.G_B2A, epoch)

            self.writeLossDataToFile(self.training_history)

            # Flush out prints each loop iteration
            sys.stdout.flush()

        self.save_model(self.D_A, epoch)
        self.save_model(self.D_B, epoch)
        self.save_model(self.G_A2B, epoch)
        self.save_model(self.G_B2A, epoch)
        print('Training finished.')

    def predict(self, test_gen, tag='pred'):
        pass

    def test(self, test_gen):
        if not self.cf.test_model:
            return
        else:
            test_save_path = join(self.cf.savepath, 'test_results')
            if not exists(test_save_path):
                makedirs(test_save_path)
            print(test_save_path)

        print('\n > Testing the model...')

        # Load best trained model

        # zzh
        start_time = time.time()
        load_epoch = self.cf.load_epoch_for_test
        self.load_model_weights(self.G_A2B, load_epoch)
        self.load_model_weights(self.G_B2A, load_epoch)
        # self.load_model_weights(self.D_A, load_epoch)
        # self.load_model_weights(self.D_B, load_epoch)

        # get correct number of test samples depending on debugging or not
        # if debug, will use n_images_test
        if self.cf.n_images_test is not None:
            nb_test_samples = self.cf.n_images_test
        else:
            samplesTestA = self.get_file_len(self.cf.testA_file_path_full)
            samplesTestB = self.get_file_len(self.cf.testB_file_path_full)
            nb_test_samples = max(samplesTestA, samplesTestB)

        test_steps = int(np.ceil(nb_test_samples / float(self.cf.batch_size_test)))
        # Evaluate model
        index = 0
        for images in test_gen:

            real_images_A_test = images[0]
            real_images_B_test = images[1]

            synthetic_images_B = self.G_A2B.predict_on_batch(real_images_A_test)
            synthetic_images_A = self.G_B2A.predict_on_batch(real_images_B_test)

            recon_images_A = self.G_B2A.predict_on_batch(synthetic_images_B)
            recon_images_B = self.G_A2B.predict_on_batch(synthetic_images_A)

            save_img(real_images_A_test[0,:,:,:], join(test_save_path, 'test_' + str(index) + '_realA.png'))
            save_img(real_images_B_test[0,:,:,:], join(test_save_path, 'test_' + str(index) + '_realB.png'))
            save_img(synthetic_images_A[0,:,:,:], join(test_save_path, 'test_' + str(index) + '_syntheticA.png'))
            save_img(synthetic_images_B[0,:,:,:], join(test_save_path, 'test_' + str(index) + '_syntheticB.png'))
            save_img(recon_images_A[0,:,:,:], join(test_save_path, 'test_' + str(index) + '_reconstructedA.png'))
            save_img(recon_images_B[0,:,:,:], join(test_save_path, 'test_' + str(index) + '_reconstructedB.png'))

            print('\n')
            print('Generate synthetic images for testing---------', index, '/', nb_test_samples)

            index += 1

            if index > test_steps:
                break

        # y_predictions = self.model.predict(test_gen.make_one_shot_iterator(), steps=nb_test_samples)
        # total_time = time.time() - start_time
        # fps = float(nb_test_samples) / total_time
        # s_p_f = total_time / float(nb_test_samples)
        # print ('   Testing time: {}. FPS: {}. Seconds per Frame: {}'.format(total_time, fps, s_p_f))
        #
        # # store predicted labels
        # result_path = join(self.cf.savepath, 'predicted_labels')
        # if exists(result_path) == False:
        #     mkdir(result_path)
        #
        # results = []
        # fp = open(self.cf.test_file_path_full)
        # image_names = fp.readlines()
        # fp.close()
        # for (idx, img_num) in enumerate(image_names):
        #     if idx > nb_test_samples-1:
        #         continue
        #     y_sample_prediction = y_predictions[idx,:,:,:,:]
        #
        #     # print('sample: ' + img_num + ', idx: ' + str(idx))
        #     # print('min: ' + str(np.min(y_sample_prediction)))
        #     # print('max: ' + str(np.max(y_sample_prediction)))
        #
        #     # compress to top probabilty
        #
        #     result = np.argmax(y_sample_prediction, axis=-1)
        #     z=np.shape(result)[-1]
        #     for i in range(z):
        #         img_num = img_num.strip('\n')
        #         save_img(result[:,:,i:i+1], join(result_path, img_num + '_slice'+str(i)+'.png'))

        print('{} pairs of synthetic images have been generated and placed in {}'
              .format(nb_test_samples, test_save_path))
        total_time = time.time() - start_time
        print('Testing time: {}'.format(total_time))
#===============================================================================
# Architecture functions

    def ck(self, x, k, use_normalization):
        # zzh: add init
        init = RandomNormal(stddev=0.02)
        x = Conv2D(filters=k, kernel_size=4, strides=2, padding='same', kernel_initializer=init)(x)
        # Normalization is not done on the first discriminator layer
        if use_normalization:
            x = InstanceNormalization(axis=3, center=True, epsilon=1e-5)(x, training=True)
        x = LeakyReLU(alpha=0.2)(x)
        return x

    def c7Ak(self, x, k):
        init = RandomNormal(stddev=0.02)
        x = Conv2D(filters=k, kernel_size=7, strides=1, padding='valid', kernel_initializer=init)(x)
        x = InstanceNormalization(axis=3, center=True, epsilon=1e-5)(x, training=True)
        x = Activation('relu')(x)
        return x

    def dk(self, x, k):
        init = RandomNormal(stddev=0.02)
        x = Conv2D(filters=k, kernel_size=3, strides=2, padding='same', kernel_initializer=init)(x)
        x = InstanceNormalization(axis=3, center=True, epsilon=1e-5)(x, training=True)
        x = Activation('relu')(x)
        return x

    def Rk(self, x0):
        init = RandomNormal(stddev=0.02)

        k = int(x0.shape[-1])
        # first layer
        x = ReflectionPadding2D((1,1))(x0)
        x = Conv2D(filters=k, kernel_size=3, strides=1, padding='valid', kernel_initializer=init)(x)
        x = InstanceNormalization(axis=3, center=True, epsilon=1e-5)(x, training=True)
        x = Activation('relu')(x)
        # second layer
        x = ReflectionPadding2D((1, 1))(x)
        x = Conv2D(filters=k, kernel_size=3, strides=1, padding='valid', kernel_initializer=init)(x)
        x = InstanceNormalization(axis=3, center=True, epsilon=1e-5)(x, training=True)
        # merge
        x = add([x, x0])
        return x

    def uk(self, x, k):
        init = RandomNormal(stddev=0.02)
        # (up sampling followed by 1x1 convolution <=> fractional-strided 1/2)
        if self.cf.use_resize_convolution:
            x = UpSampling2D(size=(2, 2))(x)  # Nearest neighbor upsampling
            x = ReflectionPadding2D((1, 1))(x)
            x = Conv2D(filters=k, kernel_size=3, strides=1, padding='valid', kernel_initializer=init)(x)
        else:
            x = Conv2DTranspose(filters=k, kernel_size=3, strides=2, padding='same', kernel_initializer=init)(x)  # this matches fractinoally stided with stride 1/2
        x = InstanceNormalization(axis=3, center=True, epsilon=1e-5)(x, training=True)
        x = Activation('relu')(x)
        return x

#===============================================================================
# Models

    def modelMultiScaleDiscriminator(self, name=None):
        x1 = Input(shape=self.cf.input_shape)
        x2 = AveragePooling2D(pool_size=(2, 2))(x1)
        #x4 = AveragePooling2D(pool_size=(2, 2))(x2)

        out_x1 = self.modelDiscriminator('D1')(x1)
        out_x2 = self.modelDiscriminator('D2')(x2)
        #out_x4 = self.modelDiscriminator('D4')(x4)

        return Model(inputs=x1, outputs=[out_x1, out_x2], name=name)

    def modelDiscriminator(self, name=None):
        # Specify input
        input_img = Input(shape=self.cf.input_shape)
        # Layer 1 (#Instance normalization is not used for this layer)
        # C64
        x = self.ck(input_img, 64, False)
        # Layer 2
        # C 128
        x = self.ck(x, 128, True)
        # Layer 3
        # C 256
        x = self.ck(x, 256, True)
        # Layer 4
        # C 512
        x = self.ck(x, 512, True)
        # Output layer
        if self.cf.use_patchgan:
            init = RandomNormal(stddev=0.02)
            x = Conv2D(filters=1, kernel_size=4, strides=1, padding='same', kernel_initializer=init)(x)
        else:
            x = Flatten()(x)
            x = Dense(1)(x)
        #x = Activation('sigmoid')(x) - No sigmoid to avoid near-fp32 machine epsilon discriminator cost
        return Model(inputs=input_img, outputs=x, name=name)

    def modelGenerator(self, name=None):
        init = RandomNormal(stddev=0.02)
        # Specify input
        input_img = Input(shape=self.cf.input_shape)
        # Layer 1
        x = ReflectionPadding2D((3, 3))(input_img)
        '''
        # zzh
        x = self.c7Ak(x, 32)
        # Layer 2
        x = self.dk(x, 64)
        # Layer 3
        x = self.dk(x, 128)

        if self.cf.use_multiscale_discriminator:
            # Layer 3.5
            x = self.dk(x, 256)

        # Layer 4-12: Residual layer
        for _ in range(4, 13):
            x = self.Rk(x)

        if self.cf.use_multiscale_discriminator:
            # Layer 12.5
            x = self.uk(x, 128)

        # Layer 13
        x = self.uk(x, 64)
        # Layer 14
        x = self.uk(x, 32)
        x = ReflectionPadding2D((3, 3))(x)
        x = Conv2D(self.cf.channel_size, kernel_size=7, strides=1)(x)
        x = Activation('tanh')(x)  # They say they use Relu but really they do not
        return Model(inputs=input_img, outputs=x, name=name)
        # zzh
        '''
        # c7s1-64
        x = self.c7Ak(x, 64)
        # Layer 2, d128
        x = self.dk(x, 128)
        # Layer 3, d256
        x = self.dk(x, 256)

        if self.cf.use_multiscale_discriminator:
            # Layer 3.5
            x = self.dk(x, 512)

        # Layer 4-12: Residual layer
        for _ in range(4, 13):
            x = self.Rk(x)

        if self.cf.use_multiscale_discriminator:
            # Layer 12.5
            x = self.uk(x, 256)

        # Layer 13, u128
        x = self.uk(x, 128)
        # Layer 14, u64
        x = self.uk(x, 64)
        x = ReflectionPadding2D((3, 3))(x)
        x = Conv2D(self.cf.channel_size, kernel_size=7, strides=1, padding='valid', kernel_initializer=init)(x)
        x = Activation('tanh')(x)  # They say they use Relu but really they do not
        return Model(inputs=input_img, outputs=x, name=name)
#===============================================================================
# Loading / Saving

    def save_model(self, model, epoch):
        # Create folder to save model architecture and weights
        directory = join(self.cf.savepath, 'saved_models')
        if not exists(directory):
            makedirs(directory)

        # model_path_w = 'saved_models/{}_weights_epoch_{}.hdf5'.format(model.name, epoch)
        model_path_w = join(directory, '{}_weights_epoch_{}.hdf5'.format(model.name, epoch))
        model.save_weights(model_path_w)
        #model_path_m = 'saved_models/{}_model_epoch_{}.json'.format(model.name, epoch)
        model_path_m = join(directory, '{}_model_epoch_{}.json'.format(model.name, epoch))
        model.save_weights(model_path_m)
        json_string = model.to_json()
        with open(model_path_m, 'w') as outfile:
            json.dump(json_string, outfile)
        print('{} has been saved in saved_models'.format(model.name))

    def writeLossDataToFile(self, history):
        keys = sorted(history.keys())
        out_file = join(self.cf.savepath, 'loss_output.csv')
        with open(out_file, 'w') as csv_file:
            writer = csv.writer(csv_file, delimiter=',')
            writer.writerow(keys)
            writer.writerows(zip(*[history[key] for key in keys]))

    #zzh
    def load_model_weights(self, model, epoch):
        # The saved model folder
        directory = join(self.cf.savepath, 'saved_models')
        model_path_w = join(directory, '{}_weights_epoch_{}.hdf5'.format(model.name, epoch))
        model.load_weights(model_path_w)

    def load_pretrained_weights(self, model):
        # The pretrained model folder
        directory = join(self.cf.savepath, 'pretrained_weights')
        # all pretrained models should be save by 'save_weights' with name of 'pretrained_xxx'.
        model_path_w = join(directory, 'pretrained_{}.hdf5'.format(model.name))
        model.load_weights(model_path_w)


"""Implementation of Rule L044."""
from typing import Dict, List

from sqlfluff.core.rules.analysis.select_crawler import SelectCrawler
from sqlfluff.core.dialects.base import Dialect
from sqlfluff.core.rules.base import BaseRule, LintResult

class RuleFailure(Exception):
    """Exception class for reporting lint failure inside deeply nested code."""

    pass

class Rule_L044(BaseRule):
    """Query produces an unknown number of result columns.

    | **Anti-pattern**
    | Querying all columns using `*` produces a query result where the number
    | or ordering of columns changes if the upstream table's schema changes.
    | This should generally be avoided because it can cause slow performance,
    | cause important schema changes to go undetected, or break production code.
    | For example:
    | * If a query does `SELECT t.*` and is expected to return columns `a`, `b`,
    |   and `c`, the actual columns returned will be wrong/different if columns
    |   are added to or deleted from the input table.
    | * `UNION` and `DIFFERENCE` clauses require the inputs have the same number
    |   of columns (and compatible types).
    | * `JOIN` queries may break due to new column name conflicts, e.g. the
    |   query references a column "c" which initially existed in only one input
    |   table but a column of the same name is added to another table.
    | * `CREATE TABLE (<<column schema>>) AS SELECT *`

    .. code-block::

        WITH cte AS (
            SELECT * FROM foo
        )

        SELECT * FROM cte
        UNION
        SELECT a, b FROM t

    | **Best practice**
    | Somewhere along the "path" to the source data, specify columns explicitly.

    .. code-block::

        WITH cte AS (
            SELECT * FROM foo
        )

        SELECT a, b FROM cte
        UNION
        SELECT a, b FROM t

    """

    _works_on_unparsable = False

    def _handle_alias(self, alias_info, dialect, queries):
        select_info_target = SelectCrawler.get(
            alias_info.from_expression_element, queries, dialect
        )
        if isinstance(select_info_target, str):
            # It's an alias to an external table whose
            # number of columns could vary without our
            # knowledge. Thus, warn.
            self.logger.debug(
                f"Query target {select_info_target} is external. Generating warning."
            )
            raise RuleFailure()
        else:
            # Handle nested SELECT.
            self._analyze_result_columns(select_info_target, dialect, queries)

    def _analyze_result_columns(
        self,
        select_info_list: List[SelectCrawler],
        dialect: Dialect,
        queries: Dict[str, List[SelectCrawler]],
    ):
        """Given info on a list of SELECTs, determine whether to warn."""
        # Recursively walk from the given query (select_info_list) to any
        # wildcard columns in the select targets. If every wildcard evdentually
        # resolves to a query without wildcards, all is well. Otherwise, warn.
        for select_info in select_info_list:
            self.logger.debug(f"Analyzing query: {select_info.select_statement.raw}")
            for wildcard in select_info.get_wildcard_info():
                if wildcard.tables:
                    for wildcard_table in wildcard.tables:
                        self.logger.debug(
                            f"Wildcard: {wildcard.segment.raw} has target {wildcard_table}"
                        )
                        # Is it an alias?
                        alias_info = select_info.find_alias(wildcard_table)
                        if alias_info:
                            # Found the alias matching the wildcard. Recurse,
                            # analyzing the query associated with that alias.
                            self._handle_alias(alias_info, dialect, queries)
                        else:
                            # Not an alias. Is it a CTE?
                            if wildcard_table in queries:
                                # Wildcard refers to a CTE. Analyze it.
                                self._analyze_result_columns(
                                    queries.pop(wildcard_table), dialect, queries
                                )
                            else:
                                # Not CTE, not table alias. Presumably an
                                # external table. Warn.
                                self.logger.debug(
                                    f"Query target {wildcard_table} is external. Generating warning."
                                )
                                raise RuleFailure()
                else:
                    # No table was specified with the wildcard. Assume we're
                    # querying from a nested select in FROM.
                    select_info_target = SelectCrawler.get(
                        select_info.select_statement, queries, dialect
                    )
                    assert isinstance(select_info_target, list)
                    self._analyze_result_columns(
                        select_info_target,
                        dialect,
                        queries,
                    )

    def _eval(self, segment, dialect, **kwargs):
        """Outermost query should produce known number of columns."""
        if segment.is_type("statement"):
            queries = SelectCrawler.gather(segment, dialect)

            # Begin analysis at the final, outer query (key=None).
            if None in queries:
                select_info = queries[None]
                try:
                    return self._analyze_result_columns(select_info, dialect, queries)
                except RuleFailure:
                    return LintResult(
                        anchor=queries[None][0].select_info.select_statement
                    )
        return None


from flask import Flask
from flask import request
from flask import Response
from flask import jsonify
import requests
import os
import json
import psycopg2

app = Flask(__name__)

players_db = os.environ['DATABASE_URL']
content_db = os.environ['HEROKU_POSTGRESQL_JADE_URL']

@app.route('/')
def sayHello():

  return "Hello there"

#API 1.1
@app.route('/game/<game_id>/player/<player_id>/character', methods=['POST'])
def createNewPlayerCharacter(game_id,player_id):
  #insert character into db

  content = request.get_json()
  title = content['title']

  with psycopg2.connect(players_db) as conn:
    cursor = conn.cursor()
    sqli_query = "INSERT INTO characters (game_id, title, player_id) VALUES (?,?,?)"
    query = cursor.execute(sqli_query,(int(game_id), title, int(player_id)))
    if query != False:#not sure if this if is needed, but i figured if the insert didn't work, I shouldn't get the lastrowid
      characters_id = cursor.lastrowid #this gets the characters_id
  #if the query failed
  if not query:
    abort(409, "Could not create character")

  return_json = {"title":title, "id":str(characters_id), "game_id":game_id, "player_id":player_id, "location":"null", "attributes":"null"}
  return jsonify(return_json), 201

#1.2 GET /player/<player_id>/character
#retrieve a list of player characters
@app.route('/player/<player_id>/character',methods=['GET'])
def get_player_characters(player_id):
  with psycopg2.connect(players_db) as conn:
    cursor = conn.cursor()
    sqli_query = "SELECT characters_id, title FROM characters WHERE player_id=?"
    cursor.execute(sqli_query, (test,))
  result = cursor.fetchall()
  final = []
  for row in result:
    item = {'id': row[0], 'title': row[1]}
    final.append(item)
  return jsonify(final), 200

#1.3 GET  /player/<player_id>/character/<character_id>
#retrieve player character details
@app.route('/player/<player_id>/character/<characters_id>',methods=['GET'])
def get_player_characters_details(player_id, characters_id):
  with psycopg2.connect(players_db) as conn:
    cursor = conn.cursor()
    sqli_query = "SELECT * FROM characters WHERE characters_id=?"
    cursor.execute(sqli_query, (characters_id,))
  result = cursor.fetchone()
  result_characters_id = result[0]
  result_game_id = result[1]
  result_player_id = result[2]
  result_title = result[3]

  with psycopg2.connect(players_db) as conn:
    cursor = conn.cursor()
    sqli_query = "SELECT * FROM characters_attributes WHERE character_id=?"
    cursor.execute(sqli_query, (characters_id,))
  result = cursor.fetchone()
  result_attributes = {"players_attributes_id":result[0],"player_id":result[1],"attr_title":result[2],"attr_value":result[3]}
  character = {"title":result_title, "id":result_characters_id, "game_id":result_game_id, "player_id":result_player_id, "attributes":result_attributes}
  return jsonify(character), 200

#get 1.4 from Laura

# 4.1 - Retrieve all items
# GET  /game/<game>/item
@app.route('/game/<game_id>/item',methods=['GET'])
def get_all_items(game_id):
  with psycopg2.connect(content_db) as conn:
    cursor = conn.cursor()
    sqli_query = "SELECT items_id, title FROM items WHERE game_id=?"
    cursor.execute(sqli_query, (game_id,))
  result = cursor.fetchall()
  return_json = []
  for row in result:
    item = {'id': row[0], 'title': row[1]}
    return_json.append(item)
  return jsonify(return_json), 200

# 4.2 add a new item to a game
# POST  /game/<game_id>/item
@app.route('/game/<game_id>/item', methods=['POST'])
def createNewItem(game_id):

  content = request.get_json()
  title = content['title']
  description = content['description']
  aliases = content['aliases']#array
  attributes = content['attributes']#object

  #3 inserts
  # 1. insert items -> title, desc, game_id
  # !!get the lastrowid
  with psycopg2.connect(content_db) as conn:
    cursor = conn.cursor()
    sqli_query = "INSERT INTO items (game_id, title, description) VALUES (?,?,?)"
    query = cursor.execute(sqli_query,(int(game_id), title, description))
    if query != False:
      item_id = cursor.lastrowid
  #if the query failed
  if not query:
    abort(409, "Could not create item")

  # 2. loop through aliases array, insert into aliases -> item_id, title
  with psycopg2.connect(content_db) as conn:
    cursor = conn.cursor()
    for v in aliases:
      sqli_query = "INSERT INTO items_aliases (item_id, title) VALUES (?,?)"
      query = cursor.execute(sqli_query, (item_id, v))
      if not query:
        abort(409, "Could not create item")

  # 3. loop through attributes object, insert items_attr. -> item_id, attr_title, attr_value
  with psycopg2.connect(content_db) as conn:
    cursor = conn.cursor()
    for k in attributes:
      sqli_query = "INSERT INTO items_attributes (item_id, attr_title, attr_value) VALUES (?,?,?)"
      query = cursor.execute(sqli_query, (item_id, k, attributes[k]))
      if not query:
        abort(409, "Could not create item")

  #return_json = {"title":title, "id":items_id, "game_id":game_id, "description":description, "aliases":aliases, "attributes":attributes}
  #return jsonify(return_json), 201
  return get_item(game_id, item_id)

# 4.3 - retrieve all item details
@app.route('/game/<game_id>/item/<item_id>',methods=['GET'])
def get_item(game_id,item_id):
  with psycopg2.connect(content_db) as conn:
    cursor = conn.cursor()
    sqli_query = "SELECT title, description FROM items WHERE game_id=? AND items_id=?"
    cursor.execute(sqli_query, (game_id, item_id))
    result = cursor.fetchone()
    title = result[0]
    description = result[1]
    sqli_query = "SELECT title FROM items_aliases WHERE item_id=?" #notice here it's item_id and above it's items_id... I was erroneously inconsistent while creating the db
    cursor.execute(sqli_query, (item_id,))
    result = cursor.fetchall()
    aliases = []
    for r in result:
      aliases.append(r[0])
    sqli_query = "SELECT attr_title, attr_value FROM items_attributes WHERE item_id=?" #notice here it's item_id and above it's items_id... I was erroneously inconsistent while creating the db
    cursor.execute(sqli_query, (item_id,))
    result = cursor.fetchall()
    attributes = {}
    for row in result:
      attributes[row[0]] = row[1]

  return_json = {"title":title, "id":item_id, "game_id":game_id, "description":description, "aliases":aliases, "attributes":attributes}
  return jsonify(return_json), 200

# 4.4 update item details
@app.route('/game/<game_id>/item/<item_id>', methods=['PUT'])
def updateItemDetails(game_id,item_id):

  content = request.get_json()
  title = content['title']
  description = content['description']
  aliases = content['aliases']#array
  attributes = content['attributes']#object

  with psycopg2.connect(content_db) as conn:
    cursor = conn.cursor()
    sqli_query = "UPDATE items SET title=?, description=? WHERE items_id=?"
    query = cursor.execute(sqli_query,(title, description,item_id))
    if not query:
      abort(409, "Could not update")
      #delete aliases and reinsert
    sqli_query = "DELETE FROM items_aliases WHERE item_id=?"
    query = cursor.execute(sqli_query,(item_id,))
    if not query:
      abort(409, "Could not update")
    for k in aliases:
      sqli_query = "INSERT INTO items_aliases (item_id, title) VALUES (?,?)"
      query = cursor.execute(sqli_query, (item_id, k,))
      if not query:
        abort(409, "Could not create item")
    #delete old attribtues and insert new ones
    sqli_query = "DELETE FROM items_attributes WHERE item_id=?"
    query = cursor.execute(sqli_query,(item_id,))
    if not query:
      abort(409, "Could not update")
     # 3. loop through attributes object, insert items_attr. -> item_id, attr_title, attr_value
    for k in attributes:
      sqli_query = "INSERT INTO items_attributes (item_id, attr_title, attr_value) VALUES (?,?,?)"
      query = cursor.execute(sqli_query, (item_id, k, attributes[k]))
      if not query:
        abort(409, "Could not create item")

  if not query:
    abort(409, "Could not update")

  return get_item(game_id,item_id)

#this method executes after every API request
@app.after_request
def after_requestuest(response):
  return response

app.debug = True
host = os.environ.get('OP', '0.0.0.0')
port = int(os.environ.get('PORT', 8080))
app.run(host=host, port=port)


from copy import deepcopy
from multiprocessing import Process, Queue
from itertools import product
import sys, os
import numpy as np
import time
import argparse

sys.path.append(os.path.abspath("."))

def kwargs_to_cmd(kwargs):
    cmd = "python main.py "
    for flag, val in kwargs.items():
        cmd += f"--{flag}={val} "

    return cmd

def run_exp(gpu_num, in_queue):
    while not in_queue.empty():
        try:
            experiment = in_queue.get(timeout=3)
        except:
            return

        before = time.time()

        experiment["multigpu"] = gpu_num
        print(f"==> Starting experiment {kwargs_to_cmd(experiment)}")
        os.system(kwargs_to_cmd(experiment))

        with open("output.txt", "a+") as f:
            f.write(
                f"Finished experiment {experiment} in {str((time.time() - before) / 60.0)}."
            )

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu-sets', default=0, type=lambda x: [a for a in x.split("|") if a])
    parser.add_argument('--seeds', default=1, type=int)
    parser.add_argument('--data', default='~/data', type=str)
    args = parser.parse_args()

    gpus = args.gpu_sets
    seeds = list(range(args.seeds))
    data = args.data

    config = "experiments/GG/splitcifar100/configs/rn18-separate-heads-randw.yaml"
    log_dir = "supermasks/runs/rn18-separate-heads-randw"
    experiments = []

    for seed in seeds:
        kwargs = {
            "config": config,
            "name": f"id=separate-heads~seed={seed}",
            "seed": seed,
            "log-dir": log_dir,
            "data": data
        }

        experiments.append(kwargs)

    print(experiments)
    input("Press any key to continue...")
    queue = Queue()

    for e in experiments:
        queue.put(e)

    processes = []
    for gpu in gpus:
        p = Process(target=run_exp, args=(gpu, queue))
        p.start()
        processes.append(p)

    for p in processes:
        p.join()

if __name__ == "__main__":
    main()


import cv2
import numpy as np
import matplotlib.pyplot as plt

img = cv2.imread('lena.jpg')
img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)

#Homogenous filter : each output is mean of its kernal neighbour
kernel = np.ones((5,5),np.float32)/25
dst =cv2.filter2D(img,-1,kernel)#Homogenous filter

blur = cv2.blur(img,(5,5)) # blur image

#Gaussian Filter. Good for high freq noise
gblur = cv2.GaussianBlur(img,(5,5),0)

#Medain filter. Replace with averge. good for salt & pepper noise
median = cv2.medianBlur(img,5)#Kernel must me odd

#bilateral filter .Preserve edges
bfilter = cv2.bilateralFilter(img,9,75,75)

titles = [ 'image','2d Convolution0','blur','GBlur','Median','bilateral']
images = [img,dst,blur,gblur,median,bfilter]

for i in range(6):
    plt.subplot(3,2,i+1)
    plt.imshow(images[i],'gray')
    plt.title(titles[i])
    plt.xticks([]),plt.yticks([])

plt.show()


from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from pytz import utc

# Flask app:
app = Flask(__name__, instance_relative_config=True)
app.config.from_object('config')
app.config.from_pyfile('config.py')

# Data base:
db = SQLAlchemy(app)
migrate = Migrate(app, db)

# Scheduler:
@app.before_first_request
def add_scheduler():
    jobstores = {
        'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')
    }
    executors = {
        'default': ThreadPoolExecutor(20),
        'processpool': ProcessPoolExecutor(5)
    }
    job_defaults = {
        'coalesce': False,
        'max_instances': 3
    }
    scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc)

    scheduler.start()

    scheduler.reschedule_job(job_id='covid_job', trigger='interval', minutes=app.config['REQUEST_INTERVAL'])
    scheduler.reschedule_job(job_id='currencies_job', trigger='interval', minutes=app.config['REQUEST_INTERVAL'])

    scheduler.print_jobs()

from app import models, routes


#
# @lc app=leetcode.cn id=1703 lang=python3
#
# [1703] 得到连续 K 个 1 的最少相邻交换次数
#
# https://leetcode-cn.com/problems/minimum-adjacent-swaps-for-k-consecutive-ones/description/
#
# algorithms
# Hard (39.19%)
# Likes:    19
# Dislikes: 0
# Total Accepted:    868
# Total Submissions: 2.2K
# Testcase Example:  '[1,0,0,1,0,1]\n2'
#
# 给你一个整数数组 nums 和一个整数 k 。 nums 仅包含 0 和 1 。每一次移动，你可以选择 相邻 两个数字并将它们交换。
#
# 请你返回使 nums 中包含 k 个 连续 1 的 最少 交换次数。
#
#
#
# 示例 1：
#
# 输入：nums = [1,0,0,1,0,1], k = 2
# 输出：1
# 解释：在第一次操作时，nums 可以变成 [1,0,0,0,1,1] 得到连续两个 1 。
#
#
# 示例 2：
#
# 输入：nums = [1,0,0,0,0,0,1,1], k = 3
# 输出：5
# 解释：通过 5 次操作，最左边的 1 可以移到右边直到 nums 变为 [0,0,0,0,0,1,1,1] 。
#
#
# 示例 3：
#
# 输入：nums = [1,1,0,1], k = 2
# 输出：0
# 解释：nums 已经有连续 2 个 1 了。
#
#
#
#
# 提示：
#
#
# 1 <= nums.length <= 10^5
# nums[i] 要么是 0 ，要么是 1 。
# 1 <= k <= sum(nums)
#
#
#

# @lc code=start
from typing import List

class Solution:
    def minMoves(self, nums: List[int], k: int) -> int:
        if k == 1:
            return 0
        n = len(nums)
        g = []
        sum = [0]
        count = -1
        for i in range(n):
            if nums[i] == 1:
                count += 1
                g.append(i - count)
                sum.append(sum[-1] + g[-1])
        m = len(g)
        ans = float('inf')
        for i in range(m - k + 1):
            mid = (i + i + k - 1) // 2
            q = g[mid]
            ans = min(ans, (2 * (mid - i) - k + 1) * q + (sum[i + k] - sum[mid + 1]) - (sum[mid]
             - sum[i]))
        return ans

# @lc code=end

1703.得到连续-k-个-1-的最少相邻交换次数.py

1703.得到连续-k-个-1-的最少相邻交换次数.py


"Based on a code from Firedrake"
"Plus moving frame -vv*(phi_x-Delta(phi)_x)"
"Half domain"
"BL: continuous Galerkin"
"rectangle domain"

from firedrake import *
from ini_eta import *
from ini_phi import *
from trans import *
from firedrake.petsc import PETSc
import numpy as np
import time

op2.init()
parameters["coffee"]["O2"] = False

# Timing
t00 = time.time()

""" ________________ Parameters ________________ """

epsilon = 0.05           # Small amplitude parameter
ep=epsilon
mu = 0.0025           # Small dispersion parameter

t=-200
dt = 0.005         # Time step
dt1=dt

""" ___________________ Mesh ___________________ """
mesh = Mesh("dom9_h4.msh")   # Load the mesh file

coords = mesh.coordinates

""" ______________ Function Space ______________ """
V = FunctionSpace(mesh, "CG", 1)    # Vector space

""" ___________ Define the functions ___________ """

eta0 = Function(V, name="eta")
phi0 = Function(V, name="phi")
eta1 = Function(V, name="eta_next")
phi1 = Function(V, name="phi_next")
eta2 = Function(V, name="eta_trans")
phi2 = Function(V, name="phi_trans")

q0 = Function(V)
q1 = Function(V)
phi_h = Function(V)
R_half = Function(V, name="eta_xx")
q_h = Function(V, name="-phi_xx")

q = TrialFunction(V)
v = TestFunction(V)

u1 = Function(V)    # eta(n)
u2 = Function(V)    # eta(n)
u3=Function(V)
u4=Function(V)
ut=Function(V)
uxt=Function(V)
d1 = Function(V)    # eta(n)

""" _____________ Initial solution _____________ """

g=.25
qq=(2/9)**(1/6)*g/np.sqrt(ep)
del1=0.001
ww=107/104
M=qq/(1+2*del1+np.sqrt(ww))
m=M*del1
k1 = -(1+.5*np.sqrt(ww))*M-m
k2 = -(.5*np.sqrt(ww))*M-m
k3 = k2+m
k4 = -k3
k5 = -k2
k6 = -k1

# Expression of eta and phi
x = SpatialCoordinate(mesh)
xx= (x[0]-5)*(4.5)**(1/6)*(ep/mu)**(1/2)
yy= x[1]*(4.5)**(1/3)*ep*(1/mu)**(1/2)
t0 = Constant(t)

eta0=initial_eta(xx,yy,u1,u2,u3,u4,ut,uxt,d1,eta0,k1,k2,k3,k4,k5,k6,t,ep,mu)
phi0=initial_phi(xx,yy,u1,d1,phi0,k1,k2,k3,k4,k5,k6,t,ep,mu)

""" _____________ Weak formulations _____________ """#,time=t

Fphi_h = ( v*(phi_h-phi0)/(0.5*dt) + 0.5*mu*inner(grad(v),grad((phi_h-phi0)/(0.5*dt)))
           + v*eta0 + 0.5*epsilon*inner(grad(phi_h),grad(phi_h))*v)*dx
phi_problem_h = NonlinearVariationalProblem(Fphi_h,phi_h)
phi_solver_h = NonlinearVariationalSolver(phi_problem_h)

# followed by a calculation of a half-step solution :math:`q`, performed using a linear solver::

aq_h = v*q*dx
Lq_h = 2.0/3.0*inner(grad(v),grad(phi_h))*dx

q_problem_h = LinearVariationalProblem(aq_h,Lq_h,q_h)
q_solver_h = LinearVariationalSolver(q_problem_h)

#
Feta = ( v*(eta1-eta0)/dt + 0.5*mu*inner(grad(v),grad((eta1-eta0)/dt))
         - 0.5*((1+epsilon*eta0)+(1+epsilon*eta1))*inner(grad(v),grad(phi_h))
         - mu*inner(grad(v),grad(q_h)))*dx
eta_problem = NonlinearVariationalProblem(Feta,eta1)
eta_solver = NonlinearVariationalSolver(eta_problem)

# and finally the second half-step (explicit this time) for the equation of :math:`\phi` is performed and :math:`q` is computed for the updated solution::
#
Fphi = ( v*(phi1-phi_h)/(0.5*dt) + 0.5*mu*inner(grad(v),grad((phi1-phi_h)/(0.5*dt)))
         + v*eta1 + 0.5*epsilon*inner(grad(phi_h),grad(phi_h))*v)*dx

phi_problem = NonlinearVariationalProblem(Fphi,phi1)
phi_solver = NonlinearVariationalSolver(phi_problem)

Lq = 2.0/3.0*inner(grad(v),grad(phi1))*dx
q_problem = LinearVariationalProblem(aq_h,Lq,q1)
q_solver = LinearVariationalSolver(q_problem)

phi1.assign(phi0)
q_solver.solve()

E_data1 = np.zeros(1)
E1 = assemble( (0.5*eta0**2+0.5*(1+epsilon*eta0)*abs(grad(phi0))**2\
                      +mu*(inner(grad(q1),grad(phi0)))+mu*( - 0.75*q1**2))*dx )
E_data1[0]=E1

PETSc.Sys.Print(t,E1)
""" _____________ Time loop _____________ """

output1 = File('data/data_tes/output.pvd')
output1.write(phi0, eta0,phi2, eta2,  time=t)

# We are now ready to enter the main time iteration loop::
t1=t
step=int(0)
T = int(200)

ddx=1/12
trans_gap=2

while t < t1+T:
      t += dt
      # PETSc.Sys.Print(t)
      phi_solver_h.solve()
      q_solver_h.solve()
      eta_solver.solve()
      phi_solver.solve()
      q_solver.solve()

      eta0.assign(eta1)
      phi0.assign(phi1)
      step +=int(1)
      if step % 50 == 0:
           output1.write(phi0, eta0,phi2, eta2,  time=t)
      if step % 400== 0:
        phi2=trans(coords,phi0,phi2,ddx,trans_gap)
        eta2=trans(coords,eta0,eta2,ddx,trans_gap)
        output1.write(phi0, eta0,phi2, eta2,  time=t)
        eta0.assign(eta2)
        phi0.assign(phi2)
        PETSc.Sys.Print(t,E1)

      E1 = assemble( (0.5*eta0**2+0.5*(1+epsilon*eta0)*abs(grad(phi0))**2\
                      +mu*(inner(grad(q1),grad(phi0)))+mu*( - 0.75*q1**2))*dx )
      E_data1=np.r_[E_data1,[E1]]

      np.savetxt('data/data_tes/energy1.csv', E_data1)

#Edata.close()
print(time.time() - t00)     # Print computational time (s)


import sys
import asyncio
import json
from utils import scan_json
import config

loop = asyncio.get_event_loop()

conn = None
log_last = 0

state = 'unknown'
uptime = None
mem_rss = None
mem_vms = None

def format_uptime(t):
    t = int(t)
    seconds = t % 60
    t = t // 60
    minutes = t % 60
    t = t // 60
    hours = t % 24
    t = t // 24
    return '{} days {}:{}:{}'.format(t, hours, minutes, seconds)

def format_mem(m):
    if m < 1024: return str(m)
    m = m / 1024
    if m < 1024: return '{:.1f}K'.format(m)
    m = m / 1024
    if m < 1024: return '{:.1f}M'.format(m)
    m = m / 1024
    return '{:.1f}G'.format(m)

def request(**req):
    if not conn: return
    conn.write(json.dumps(req).encode())

class ClientProtocol(asyncio.Protocol):
    def connection_made(self, transport):
        global conn
        conn = transport
        request(method="set_property", key="keep_connection", value=True)
        request(method="set_property", key="push_notifications", value=True)
        request(method="refresh", count=10)
        self.buffer = b'';
        print("connected")

    def connection_lost(self, exc):
        loop.stop()
        print("disconnected")

    def data_received(self, data):
        self.buffer += data
        while True:
            (res, been_read, error) = scan_json(self.buffer)
            if error:
                print(error)
            self.buffer = self.buffer[been_read:]
            if not res: break
            if res['status'] == 'ok':
                if 'log' in res:
                    global log_last
                    for ent in res['log']:
                        sys.stdout.write(ent['message'])
                        if ent['time'] > log_last: log_last = ent['time']
                if 'state' in res:
                    global state, uptime, mem_rss, mem_vms
                    state = res['state']
                    uptime = res.get('uptime', None)
                    mem_rss = res.get('mem_rss', None)
                    mem_vms = res.get('mem_vms', None)
            else:
                print("error: "+res['error'])

def keyboard_reader():
    line = sys.stdin.readline().strip()
    if conn:
        if len(line) == 0: return
        if line == ':help' or line == ':?':
            print('console commands:')
            print(':log, :refresh  - send refresh request')
            print(':status, :state - show process state')
            return
        elif line == ':log' or line == ':refresh':
            request(method="refresh", time=log_last)
            return
        elif line == ':status' or line == ':state':
            print('state = '+state)
            if (uptime): print('uptime = '+format_uptime(uptime))
            if (mem_rss): print('mem_rss = '+format_mem(mem_rss))
            if (mem_vms): print('mem_vms = '+format_mem(mem_vms))
            return
        if line[0] == ':':
            request(method="daemon_command", command=line[1:])
        else:
            request(method="process_command", command=line)

loop.add_reader(sys.stdin, keyboard_reader)
try:
    loop.run_until_complete(
        loop.create_unix_connection(ClientProtocol, config.daemon_socket)
    );
except FileNotFoundError:
    print(config.daemon_socket+" doesn't exist")
    exit()

try:
    loop.run_forever()
except KeyboardInterrupt:
    print("\nKeyboardInterrupt")
    pass

loop.close()


import time
import IEM as iem
import pickle as pkl

# Classes
class Paquete:
    """Clase que define a el paquete de venta"""
    def __init__(self, nombre, destino, precio):
        self.nombre = nombre
        self.destino = destino
        self.precio = precio

# Functions

def cargar_paquetes():
    try:
        paquetes = pkl.load(open("Paquetes.data", "rb"))
    except FileNotFoundError:
        paquetes = []
        pkl.dump(paquetes,open("Paquetes.data", "wb"))
    finally:
        return paquetes

def cargar_libro():
    try:
        libro = pkl.load(open("libro.data", "rb"))
    except FileNotFoundError:
        libro = []
        pkl.dump(libro,open("libro.data", "wb"))
    finally:
        return libro

def nuevo_paquete():
    nuevo_nombre = input("Como se llama el paquete? ")
    nuevo_destino = input("Cual es el destino del paquete? ")
    try:
        nuevo_precio = int(input("Cual es el precio del paquete? "))
    except ValueError:
        print("El precio necesita ser un numero, intenta nuevamente...")
        nuevo_precio = int(input("Precio: "))
    pack = Paquete(nuevo_nombre, nuevo_destino, nuevo_precio)
    paquetes.append(pack)
    pkl.dump(paquetes, open("Paquetes.data", "wb"))

def leer_paquetes():
    for pack in paquetes:
        print("Nombre: {}\nDestino: {} \nPrecio: {}".format(pack.nombre, pack.destino, pack.precio))

pause = iem.pause
clear = iem.cls

paquetes = cargar_paquetes()
libro = cargar_libro()

leer_paquetes()


import pandas as pd
import numpy as np
import re

"""
This module aims at separating the different column types
"""

TEXT_THRESHOLD = 10
CATS_THRESHOLD = 10

def types(df, verbose=True):
    text, tags, cats, boolean= [], [], [], []
    cols = df.select_dtypes(exclude=[np.number]).columns.values
    N = max(len(col) for col in cols)
    for col in df.select_dtypes(exclude=[np.number]).columns:
        if df[col].dtype == np.bool:
            l = 1
        else:
            l = df[col].astype(str).fillna('None').str.split().str.len().max()
        n = df[col].nunique()
        if n == 2:
            boolean.append(col)
            if verbose:print(col.ljust(N, ' '), '-> bool', f'(max words: {l}, unique: {n})')
        elif n < CATS_THRESHOLD:
            if verbose:print(col.ljust(N, ' '), '-> cats', f'(max words: {l}, unique: {n})')
            cats.append(col)
        else:
            if l < TEXT_THRESHOLD:
                if verbose:print(col.ljust(N, ' '), '-> tags', f'(max words: {l}, unique: {n})')
                tags.append(col)
            else:
                if verbose:print(col.ljust(N, ' '), '-> text', f'(max words: {l}, unique: {n})')
                text.append(col)

    return {
        'num': list(df.select_dtypes(include=[np.number]).columns.values),
        'text': text,
        'tags': tags,
        'cats': cats,
        'bool': boolean
    }

first_cap_re = re.compile('(.)([A-Z][a-z]+)')
all_cap_re = re.compile('([a-z0-9])([A-Z])')
def to_snake_case(name):
    s1 = first_cap_re.sub(r'\1_\2', name)
    return all_cap_re.sub(r'\1_\2', s1).lower()

def downcast(df):
    d = df.copy()
    for col in df:
        n = d[col].nunique()
        if n < 2:
            d.drop(col,inplace=True,axis=1)
            continue
        if n == 2:
            if df[col].dtype == np.bool:
                continue
            a = df[col].iloc[0]
            d[col] = d[col] == a
            continue
        if d[col].dtype == np.int:
            d[col] = pd.to_numeric(d[col],downcast='integer')
            d[col] = pd.to_numeric(d[col],downcast='unsigned')
            continue
        if d[col].dtype == np.float:
            try:
                if (df[col]==df[col].astype(int)).all():
                    d[col] = pd.to_numeric(d[col],downcast='integer')
                    d[col] = pd.to_numeric(d[col],downcast='unsigned')
            except:
                pass
            continue
        if d[col].dtype == 'object':
            if 2 * n <= len(d.index) :
                d[col] = d[col].fillna('MISSING').astype('category')
    return d


import heapq
import re
import sys

import math

def count_upper_decile(filename):
    pattern = re.compile(b"(\d+) usec")
    count = 0

    with open(filename, 'rb') as f:
        for line in f:
            if line.startswith(b"open"):
                break

        for line in f:
            if line.startswith(b"open"):
                count += 1

        if count > 0:
            heap_size = int(math.ceil(count / 10))
            heap = [0] * heap_size

            f.seek(0)
            for line in f:
                if line.startswith(b"open"):
                    break

            for line in f:
                if line.startswith(b"open"):
                    value = int(pattern.search(line).group(1))
                    if value > heap[0]:
                        heapq.heappushpop(heap, value)
            return heap[0]
        else:
            return None

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Name of file was not found.", file=sys.stderr)
    else:
        try:
            upper_decile = count_upper_decile(sys.argv[1])
            if upper_decile is None:
                print("Sample is empty")
            else:
                print("upper decile = {}".format(upper_decile))
        except Exception as e:
            print(e, file=sys.stderr)


"""
 Author: Prava Dhulipalla
 Retrive sentences from the text that have the candidate name(s) passed

"""

import nltk

# list of words for scoring positive/negative
# NOTE: need to add more words
pos_words = [
    'good',
    'best',
    'great',
    'greatest'
    'excellent',
    'positive',
    'tremendous',
    'marvelous',
    'amazing',
    'wonderful',
    'fabulous',
    'truthful',
    'decent',
    'improve',
    'intelligent',
    'smart',
    'honest'
    'nice',
    'fine',
    'love',
    'loves'
    ]

neg_words = [
    'bad',
    'worse',
    'mean',
    'crooked',
    'liar',
    'fraud',
    'stupid',
    'nasty',
    'ruin',
    'destroy',
    'racist',
    'racists',
    'bigot',
    'bigots',
    'extremist',
    'extremists',
    'terrorist',
    'terrorists',
    'negative',
    'fringe',
    'hate',
    'hates',
    'hatred',
    'dishonest',
    'bombastic',
    'sad',
    'sick',
    'deplorable',
    'deplorables'
    ]

class Quote:
    """
    A quote and a positive/negative/neutral flag
    """
    def __init__(self):
        self.text = ''
        self.tone = ''

    def __str__(self):
        return self.text + ' (' + self.tone + ')'

def get_quote(text):
    """
    We use a list of positive and negative words to identify the tone.
    If positive word exists in the sentence, but not "not or no" + word, we add 1 to the score.
    If negative word exists in the sentence, but not "not or no" + word, we subtract 1 from the score.
    positive score indicates positive tone, negative score indicates negative tone and zero score indicates neutral
    """
    articles = ['a', 'an', 'the']
    negatives = ['no', 'not']
    quote = Quote()
    quote.text = text
    text_words = nltk.word_tokenize(text)
    score = 0

    for index, word in enumerate(text_words):
        if word.lower() in pos_words:
            # logic to take care of phrases like "not good" or "not a good person"
            if ((index > 0 and text_words[index-1].lower() in negatives) or
                    (index > 1 and text_words[index-1].lower() in articles and
                     text_words[index-2].lower() in negatives)):
                score -= 1          # not positive = negative
            else:
                score += 1          # postive

        elif word.lower() in neg_words:
            # logic to take care of phrases like "not bad" or "not a bad person"
            if ((index > 0 and text_words[index-1].lower() in negatives) or
                    (index > 1 and text_words[index-1].lower() in articles and
                     text_words[index-2].lower() in negatives)):
                score += 1          # not negative = positive
            else:
                score -= 1          # negative

    if score > 0:
        quote.tone = 'positive'
    elif score < 0:
        quote.tone = 'negative'
    else:
        quote.tone = 'neutral'

    return quote

def get_sentences(text, names):
    """
    Returns sentences from text that conains candidate names(s).
    names is a list of name variations of candindate like ['Hillary', 'Clinton', 'Hillary Clinton']
    or ['Donald', 'Trump', 'Donald J. Trump']
    """

    sentences1 = nltk.sent_tokenize(text)
    sentences2 = []
    for sentence in sentences1:
        if any(name in sentence for name in names):
            sentences2.append(sentence)

    return sentences2

if __name__ == "__main__":
    names = ['William', 'Derksen', 'Will']
    text = "I told Will to shut up. Will got hacked. Will doesn't know how to listen to directions. Kill Will."
    s = get_sentences(text, names)
    for x in s:
        print(get_quote(x))


import logging
from logging.handlers import TimedRotatingFileHandler
import os
import json
from collections import OrderedDict
from datetime import datetime

from .mylog import log

CHECKER_LOG_DIR = "/logs"
CHECKER_STATUS_DIR = "/status"

def get_checker_log_dir():
    return CHECKER_LOG_DIR

def get_checker_status_dir():
    return CHECKER_STATUS_DIR

_checker_loggers = dict()

def get_checker_logger(checker_id):
    if checker_id in _checker_loggers:
        return _checker_loggers[checker_id]
    checker_logger = logging.getLogger(checker_id)
    checker_logger.setLevel(logging.INFO)
    checker_logger.propagate = False
    handler = TimedRotatingFileHandler(
        filename=os.path.join(get_checker_log_dir(),
                              "{0:s}.log".format(checker_id)),
        when="midnight",
        backupCount=10,
        utc=True)
    handler.setFormatter(logging.Formatter("%(message)s"))
    checker_logger.addHandler(handler)
    _checker_loggers[checker_id] = checker_logger
    return checker_logger

def get_checker_writer(checker_id):
    return open(os.path.join(get_checker_status_dir(),
                             "{0:s}.json".format(checker_id)),
                "w")

def _checker_result_record(checker_id, checker_result):
    return OrderedDict([
        ("timestamp", datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")),
        ("checker_id", checker_id),
        ("result", checker_result._asdict())
    ])

def write_checker_log(checker_id, checker_result):
    logger = get_checker_logger(checker_id)
    logger.info(json.dumps(_checker_result_record(checker_id, checker_result)))

def write_checker_status(checker_id, checker_result):
    with get_checker_writer(checker_id) as w:
        w.write(json.dumps(
            _checker_result_record(checker_id, checker_result),
            indent=2
        ))
        w.write('\n')
        w.flush()
        os.fsync(w.fileno())

_checker_memlog = dict()

def write_checker_memlog(checker_id, checker_result):
    _checker_memlog[checker_id] = checker_result

def get_checker_memlog(checker_id):
    if not checker_id in _checker_memlog:
        log.debug("Memlog for checker %s not found.", checker_id)
    return _checker_memlog.get(checker_id)

def log_checker_result(checker_id, checker_result):
    write_checker_memlog(checker_id, checker_result)
    write_checker_status(checker_id, checker_result)
    write_checker_log(checker_id, checker_result)


# -*- coding:utf-8 -*-
import os.path
from selenium import webdriver
from common import logger
from config import config_info

log = logger.Logger(logger="BrowserEngine").getlog()

class BrowserEngine(object):
    '''
    3.os.path.dirname(path)
    返回path的目录。其实就是os.path.split(path)的第一个元素。
    dir = F:\web_automation
    '''
    dir = os.path.dirname(os.path.abspath('.'))
    chrome_driver_path = dir + '/tools/chromedriver.exe'
    firefox_driver_path = dir + '/tools/geckodriver.exe'
    ie_driver_path = dir + '/tools/IEDriverServer.exe'

    def __init__(self,driver):
        self.driver = driver

    def open_browser(self):
        browser = config_info.GetConfigData().getconfigdata()['browsername']
        url = config_info.GetConfigData().getconfigdata()['URL']

        if browser == 'Chrome':
            # self.driver = webdriver.Chrome()
            self.driver = webdriver.Chrome(self.chrome_driver_path)
            log.info("Starting chrome browser.")
        elif browser == 'Firefox':
            self.driver = webdriver.Firefox(self.firefox_driver_path)
            log.info("Starting firefox browser.")
        elif browser == 'PhantomJS':
            self.driver = webdriver.PhantomJS()
            log.info("Starting PhantomJS browser.")
        elif browser == 'Ie':
            self.driver = webdriver.Ie(self.ie_driver_path)
            log.info("Starting ie browser.")

        self.driver.get(url)
        log.info("Open url: %s" % url)
        self.driver.maximize_window()
        log.info("Maximize the current window.")
        self.driver.implicitly_wait(10)
        log.info("Set implicitly wait 10 seconds.")
        return self.driver

if __name__ == '__main__':
    browser = config_info.GetConfigData().getconfigdata()['browsername']
    url = config_info.GetConfigData().getconfigdata()['URL']
    print(browser)
    print(url)


"""empty message

Revision ID: 68ac6fb1c086
Revises: 9f2abdd7a490
Create Date: 2018-06-25 12:49:54.650669

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '68ac6fb1c086'
down_revision = '9f2abdd7a490'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('department_tree',
    sa.Column('id', mysql.BIGINT(unsigned=True), nullable=False),
    sa.Column('department_id', mysql.BIGINT(unsigned=True), nullable=False),
    sa.Column('parent_id', mysql.BIGINT(unsigned=True), nullable=False),
    sa.Column('depth', mysql.BIGINT(unsigned=True), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    mysql_charset='utf8mb4',
    mysql_engine='InnoDB'
    )
    op.create_index('idx_department_tree_d', 'department_tree', ['department_id'], unique=False)
    op.create_index('idx_department_tree_p', 'department_tree', ['parent_id'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('idx_department_tree_p', table_name='department_tree')
    op.drop_index('idx_department_tree_d', table_name='department_tree')
    op.drop_table('department_tree')
    # ### end Alembic commands ###


import numpy as np
import tensorflow as tf
from dataset import read_libsvm, Dataset
from softmax_model import placeholder, variables, inferDense, loss_train, calAcc

def train_save(ds: Dataset, t_labs: np.ndarray, t_data: np.ndarray, num_feats: int, num_class: int):
    x_data, labels = placeholder(num_feats, num_class)
    weight, bias, global_steps = variables(num_feats, num_class)
    logits, predict = inferDense(x_data, weight, bias)
    acc = calAcc(labels, predict)
    train_op, loss = loss_train(labels, logits, global_steps)
    tf.summary.scalar('acc', acc)
    tf.summary.scalar('loss', loss)
    init = tf.global_variables_initializer()

    merged = tf.summary.merge_all()
    with tf.Session() as sess:
        sess.run(init)
        writer = tf.summary.FileWriter('logs', sess.graph)

        for y_batch, x_batch in ds:
            _, gs = sess.run(fetches=[train_op, global_steps], feed_dict={
                x_data: x_batch, labels: y_batch
            })

            if gs % 200 == 0:
                acc_val, summary = sess.run(fetches=[acc, merged], feed_dict={
                    x_data: t_data, labels: t_labs
                })
                print(type(summary))
                writer.add_summary(summary, gs)

                print(acc_val)
        writer.close()

if __name__ == '__main__':
    n_feats, batch_size, n_class = 180, 128, 3
    tr_data = read_libsvm("data/dna.scale.tr")
    test_labs, test_data = read_libsvm("data/dna.scale.t", one_hot=True)
    train_ds = Dataset(tr_data, nclass=n_class, max_epoch=1000, batch_size=batch_size)

    train_save(train_ds, test_labs, test_data, num_feats=n_feats, num_class=n_class)


#!/usr/bin/python3

import argparse
import collections
import functools
import itertools
import re
import sys
import typing

def load_file(filename):
  contents = []
  with open(filename, 'r') as f:
    for line in f:
      line = line.strip() # remove trailing newline
      contents.append(line)
  return contents

def load_groups(filename):
  # because some files follow this format instead.
  contents = []
  acc = []
  for line in load_file(filename):
    if line:
      acc.append(line)
    else:
      contents.append(acc)
      acc = []

  if acc:
    contents.append(acc)

  return contents

def any_item(iterable, default=None):
  for i in iterable:
    return i
  return default

# re-implement itertools.product to better handle our use case
# only handles length
def specialized_product(*args):
  l = len(args)
  assert l == 1 or l == 2
  if l == 1:
    for a in args[0]:
      yield a
    return
  for a in args[0]:
    for b in args[1]:
      yield a + b

def grouper(string, n):
  l = len(string)
  assert l % n == 0
  out = []
  for i in range(l // n):
    offset = i * n
    sub = string[offset:offset + n]
    out.append(sub)
  return out

class MessageParser:
  @staticmethod
  def __rule_match(line):
    s = line.split(" ")
    return tuple([int(i) for i in line.split(" ")])

  @staticmethod
  def from_lines(lines):
    # the dict has the following structure:
    # [int] = list(value)
    # value = str | tuple(int+)
    rules = {}
    for line in lines:
      key, match = line.split(": ")
      key = int(key)
      if "\"" in match:
        rules[key] = [match[1:-1]]
      elif "|" in match:
        m1, m2 = match.split(" | ")
        rules[key] = [MessageParser.__rule_match(m1), MessageParser.__rule_match(m2)]
      else:
        rules[key] = [MessageParser.__rule_match(match)]

    return MessageParser(rules)

  def __init__(self, rules):
    self.rules = rules
    self.matches = None

  def __repr__(self):
    return "MessageParser({})".format(repr(self.rules))

  def __matching(cache, rules, num):
    if num in cache:
      return cache[num]

    # generate all the matches for this rule.
    assert type(num) == int
    #print("generating", num, flush=True)

    rule = rules[num]

    # every rule is a list of matching things.
    matches = []
    for i, subrule in enumerate(rule):
      if type(subrule) == str:
        #print("str match", "{}.{}".format(num, i), subrule, flush=True)
        matches.append(subrule)
      else:
        submatches = [MessageParser.__matching(cache, rules, r) for r in subrule]
        #print("submatch", "{}.{}".format(num, i), submatches, flush=True)
        expanded = specialized_product(*submatches)
        #print("expanded", "{}.{}".format(num, i), expanded, flush=True)
        matches.extend(expanded)

    #print("generated", num, "value", matches, "from", rules[num], flush=True)
    cache[num] = matches
    return cache[num]

  def __gen_matches(self):
    # generate all matches.
    cache = {}
    assert self.rules[0] == [(8, 11)]
    assert self.rules[8] == [(42,)]
    assert self.rules[11] == [(42, 31)]
    # This evaluates almost everything, and these rules are needed later.
    MessageParser.__matching(cache, self.rules, 31)
    MessageParser.__matching(cache, self.rules, 42)
    # convert rules 42 and 31 into sets, because we perform a bunch of contains
    # checks with those in particular.
    cache[42] = frozenset(cache[42])
    cache[31] = frozenset(cache[31])

    matches = None
    if False:
      # Generating this takes a considerable amount of time, that's not worth it.
      #
      # 1) generating all the values for rule 0 is slow: ~330ms
      # 2) converting the list to a set is slow: ~550ms
      #
      # Lookups in the set aren't slow, but amortized over construction time, it's not really worth it.
      #
      # Using the optimized 3 set lookup, we avoid ~880ms of overhead. See MessageParser.part1_match
      #
      # aside) not converting to a set and doing lookups is really slow: ~8s

      MessageParser.__matching(cache, self.rules, 0)
      matches = frozenset(cache[0])
      #matches = cache[0]
      # remove rules 0, 8, and 11 from the cache. They are changed in part 2
      # and we don't want to accidentally rely on them for part 1.
      del cache[0]
      del cache[8]
      del cache[11]

    # We take advantage of this property as well.
    # check it only once.
    p = len(any_item(cache[42]))
    for r in cache[42]:
      assert p == len(r)
    for r in cache[31]:
      assert p == len(r)

    # making the matching values into a set speeds up all contains checks.
    # 200% worth it.
    return (cache, matches)

  def gen_cache(self):
    assert self.rules[0] == [(8, 11)]
    assert self.rules[8] == [(42,)]
    assert self.rules[11] == [(42, 31)]

    if self.matches is None:
      self.matches = self.__gen_matches()

  def chunk_size(self):
    self.gen_cache()
    rule42 = self.matches[0][42]
    return len(any_item(rule42))

  def count(self):
    self.gen_cache()
    c42 = self.matches[0][42]
    c31 = self.matches[0][31]
    return len(c42) * len(c42) * len(c31)

  def part1_match(self, value):
    # This strategy is _much_ faster than evaluating all the strings that match rule 0
    # and checking if the string we are passed is in that set.
    self.gen_cache()

    # were going to abuse the rule structure,
    # make sure it matches our expectations.
    assert self.rules[0] == [(8, 11)]
    assert self.rules[8] == [(42,)]
    assert self.rules[11] == [(42, 31)]
    rule31 = self.matches[0][31]
    rule42 = self.matches[0][42]

    chunk_size = self.chunk_size()
    if len(value) % chunk_size != 0 or len(value) // chunk_size != 3:
      return False

    chunked = grouper(value, chunk_size)

    # if you noticed above:
    # rule 0 = 8 11
    #        = 42 42 31
    return chunked[0] in rule42 and chunked[1] in rule42 and chunked[2] in rule31

  def part2_match(self, value):
    self.gen_cache()
    rule42 = self.matches[0][42]
    rule31 = self.matches[0][31]

    # They have overlap.
    #assert len(rule42.union(rule31)) == 0

    # The new 8 & 11 rules:
    #
    # 8: 42 | 42 8
    # 11: 42 31 | 42 11 31
    #
    # equivalent (ish):
    #
    # 8 can be seen as: 42 +
    # 11 is approximately matching brackets
    #
    # this makes rule 0:
    # 42 + 42 {n} 31 {n}

    # since everything in rule42 and rule31 has the same length we can chunk the input. :)
    chunk_size = self.chunk_size()
    if len(value) % chunk_size != 0:
      return False

    chunked = grouper(value, chunk_size)
    #print("chunked", chunked)

    # recall we are trying to match:
    #
    # 42+ 42{n} 31{n}
    #
    # Rule 11 looks like bracket matching, but because we have rule 8
    # we can greedily match rule 42, and once we stop matching it, the rest
    # must match rule 31. However (!) we must make sure that we have more
    # matches against rule 42 than 31, because rule 8 requires at least 1
    # match.
    #
    # Therefore we end up with:
    #
    # 42{x} 31{y} ; x > y

    i = 0 # for
    for i in range(len(chunked)):
      # i isn't shadowed, it updates
      if chunked[i] not in rule42:
        break

    count42 = i

    for i in range(i, len(chunked)):
      if chunked[i] not in rule31:
        return False

    return count42 > len(chunked) - count42

def parseit(things):
  return (MessageParser.from_lines(things[0]), things[1])

def part1(things):
  parser, messages = things

  count = 0
  for m in messages:
    #print("checking", m)
    if parser.part1_match(m):
      #print("matches!")
      count += 1

  return count

def part2(things):
  parser, messages = things

  # part2 changes 2 rules at the root of matching stuff.
  print("42:", len(parser.matches[0][42]))
  print("31:", len(parser.matches[0][31]))

  count = 0
  for m in messages:
    #print("checking", m)
    if parser.part2_match(m):
      #print("matches!")
      count += 1

  return count

def main(filename):
  things = parseit(load_groups(filename))

  #print(things[0])
  print("matching strings (part 1):", things[0].count())
  print()

  print("part 1:", part1(things))
  print("part 2:", part2(things))

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument('input', nargs='?', default='/dev/stdin')

  args = parser.parse_args(sys.argv[1:])
  main(args.input)


#! /usr/bin/env python

import os
import unittest
from types import GeneratorType

from grader import Grader, points
from hw0 import case_sarcastically, detokenize, gen_sentences

class TestGenSentences(unittest.TestCase):
    @points(5)
    def test_type(self) -> None:
        """Test that a generator is returned."""
        gen = gen_sentences(os.path.join("test_data", "hw0_tokenized_text_1.txt"))
        self.assertEqual(type(gen), GeneratorType)

    @points(10)
    def test_basic(self) -> None:
        """Test reading a basic file."""
        gen = gen_sentences(os.path.join("test_data", "hw0_tokenized_text_1.txt"))
        self.assertEqual(
            next(gen), ["Tokenized", "text", "is", "easy", "to", "work", "with", "."]
        )
        self.assertEqual(
            next(gen), ["Writing", "a", "tokenizer", "is", "a", "pain", "."]
        )
        with self.assertRaises(StopIteration):
            next(gen)

    @points(10)
    def test_advanced(self) -> None:
        """Test reading a more complex file."""
        gen = gen_sentences(os.path.join("test_data", "hw0_tokenized_text_2.txt"))
        self.assertEqual(next(gen), ["Hello", ",", "world", "!"])
        # Between these sentences, there is a line in the file with a single space,
        # which should be skipped over.
        self.assertEqual(next(gen), ["This", "is", "a", "normal", "sentence", "."])
        self.assertEqual(
            next(gen),
            [
                '"',
                "I",
                "don't",
                "like",
                "it",
                "when",
                "there's",
                "too",
                "much",
                "punctuation",
                "!",
                '"',
                ",",
                "they",
                "exclaimed",
                ".",
            ],
        )
        with self.assertRaises(StopIteration):
            next(gen)

class TestDetokenize(unittest.TestCase):
    @points(10)
    def test_simple(self) -> None:
        """Test a simple sentence."""
        self.assertEqual(detokenize(["Hello", ",", "world", "!"]), "Hello, world!")

    @points(5)
    def test_quotes1(self) -> None:
        """Test how quotes are handled."""
        self.assertEqual(
            detokenize(
                [
                    '"',
                    "I",
                    "don't",
                    "know",
                    "what",
                    "NLP",
                    "is",
                    ",",
                    '"',
                    "he",
                    "said.",
                ]
            ),
            '"I don\'t know what NLP is," he said.',
        )
        self.assertEqual(
            detokenize(
                ['"', "Too", "much", "punctuation", "!", '"', "they", "exclaimed", "."]
            ),
            '"Too much punctuation!" they exclaimed.',
        )

    @points(5)
    def test_quotes2(self) -> None:
        """Test how quotes are handled."""
        self.assertEqual(
            detokenize(
                [
                    "She",
                    "said",
                    ",",
                    '"',
                    "I",
                    "don't",
                    "like",
                    "punctuation",
                    ",",
                    "do",
                    "you",
                    "?",
                    '"',
                ]
            ),
            'She said, "I don\'t like punctuation, do you?"',
        )

    @points(5)
    def test_strange_punc(self) -> None:
        """Test unusual punctuation."""
        self.assertEqual(
            detokenize(
                [
                    "Punctuation",
                    "can",
                    "surprise",
                    "you",
                    ";",
                    "no",
                    "one",
                    "expects",
                    "the",
                    "interrobang",
                    "‽",
                ]
            ),
            "Punctuation can surprise you; no one expects the interrobang‽",
        )

    @points(5)
    def test_em_dash(self) -> None:
        """Test em dash."""
        self.assertEqual(
            detokenize(
                [
                    "The",
                    "em",
                    "dash",
                    "—",
                    "one",
                    "of",
                    "my",
                    "favorite",
                    "characters",
                    "—",
                    "is",
                    "often",
                    "mistaken",
                    "for",
                    "the",
                    "en",
                    "dash",
                    ".",
                ]
            ),
            "The em dash—one of my favorite characters—is often mistaken for the en dash.",
        )

class TestSarcasticCaser(unittest.TestCase):
    @points(5)
    def test_no_punc(self) -> None:
        """Test basic text."""
        assert case_sarcastically("hello") == "hElLo"

    @points(10)
    def test_punc1(self) -> None:
        """Test how punctuation is handled."""
        assert case_sarcastically("hello, friend!") == "hElLo, FrIeNd!"

    @points(10)
    def test_punc2(self) -> None:
        """Test how punctuation is handled."""
        assert case_sarcastically('Say "hello," friend‽') == 'sAy "HeLlO," fRiEnD‽'

def main() -> None:
    tests = [
        TestGenSentences,
        TestDetokenize,
        TestSarcasticCaser,
    ]
    grader = Grader(tests)
    grader.print_results()

if __name__ == "__main__":
    main()