# -*- coding: utf-8 -*-
"""
Created on Thu June 26 11:00:10 2020

@author: Gerardo Cervantes

Purpose: This will handle having both the generator and discriminator, as well as training the models given the data
"""

import torch
from src.data import data_load
from src.losses.loss_functions import supported_loss_functions, supported_losses
from src.optimizers.optim_functions import supported_optimizer_list, optimizer_factory
from torch_ema import ExponentialMovingAverage
from torchinfo import summary
from src.data.data_load import normalize, unnormalize
from src.utils import os_helper
import os

class GanModel:

    def __init__(self, generator, discriminator, num_classes, accelerator, torch_dtype, model_arch_config, train_config):

        if train_config.getboolean('channels_last'):
            discriminator.set_channels_last()
            generator.set_channels_last()

        self.num_classes = num_classes
        self.device = accelerator.device
        generator = generator.to(self.device)
        discriminator = discriminator.to(self.device)
        self.torch_dtype = torch_dtype
        self.latent_vector_size = int(model_arch_config['latent_vector_size'])
        beta1 = float(train_config['beta1'])
        beta2 = float(train_config['beta2'])
        generator_wd = float(train_config['generator_wd'])
        discriminator_wd = float(train_config['discriminator_wd'])
        generator_lr = float(train_config['generator_lr'])
        discriminator_lr = float(train_config['discriminator_lr'])
        self.orthogonal_value = float(model_arch_config['orthogonal_value'])
        loss_type = model_arch_config['loss_type']
        self.is_omni_loss = 'omni-loss' == loss_type.lower()
        self.accumulation_iterations = int(train_config['accumulation_iterations'])
        self.mixed_precision = train_config.getboolean('mixed_precision')
        self.batch_iterations = 0

        self.criterion, self.fake_label, self.real_label = supported_loss_functions(train_config['loss_function'])
        if self.criterion is None:
            raise ValueError("Loss given not found: " + train_config['loss_function'] +
                             "\nPossible options are: " + str(supported_losses()))
        gan_optimizer = optimizer_factory(train_config['optimizer'])
        if not gan_optimizer:
            raise ValueError("Optimizer given not found: " + train_config['optimizer'] +
                             "\nPossible options are: " + str(supported_optimizer_list()))
        # Setup Adam optimizers for both G and D
        self.optimizerD = gan_optimizer(discriminator.parameters(), lr=discriminator_lr, betas=(beta1, beta2),
                                     weight_decay=discriminator_wd)
        self.optimizerG = gan_optimizer(generator.parameters(), lr=generator_lr, betas=(beta1, beta2),
                                     weight_decay=generator_wd)
        discriminator.zero_grad()
        generator.zero_grad()

        # Set EMA
        ema_enabled = model_arch_config.getboolean('generator_ema')
        ema_decay = model_arch_config['ema_decay']
        self.ema = ExponentialMovingAverage(generator.parameters(),
                                            decay=float(ema_decay)) if ema_enabled else None

        self.netD, self.netG = discriminator, generator
        self.accelerator = accelerator

    def create_real_labels(self, b_size, labels):
        if self.is_omni_loss:
            # Real_label is of size [b_size, num_classes]
            real_label = torch.nn.functional.one_hot(labels, num_classes=self.num_classes)
            # Adds 2 columns, One with all values 1, the second with all values 0. Size [b_size, num_classes + 2]
            real_label = torch.concat((real_label, torch.ones(b_size, 1, device=self.device, dtype=self.torch_dtype),
                                    torch.zeros(b_size, 1, device=self.device, dtype=self.torch_dtype)), dim=1)
            # Replace the 0s and 1s with value of the fake_label, or true_label
            real_label[real_label == 0] = self.fake_label
            real_label[real_label == 1] = self.real_label
        else:
            real_label = torch.full((b_size,), self.real_label, dtype=self.torch_dtype, device=self.device)
        return real_label

    def create_fake_labels(self, b_size):
        if self.is_omni_loss:
            fake_label = torch.full((b_size, self.num_classes + 1), self.fake_label, dtype=self.torch_dtype, device=self.device)
            fake_label = torch.concat((fake_label, torch.ones(b_size, 1, device=self.device, dtype=self.torch_dtypee)), dim=1)
        else:
            fake_label = torch.full((b_size,), self.fake_label, dtype=self.torch_dtype, device=self.device)
        return fake_label

    def train_step(self, batches_accumulated):
        self.netG.train()
        self.netD.train()
        discriminator_loss, accum_labels = self.discriminator_step(batches_accumulated)
        generator_loss = self.generator_step(accum_labels)
        if self.ema:
            self.ema.update()
        self.netD.requires_grad_(requires_grad=True)
        return discriminator_loss, generator_loss

    def discriminator_step(self, batches_accumulated):

        self.netD.requires_grad_(requires_grad=True)
        self.optimizerD.zero_grad(set_to_none=True)
        accum_labels = []
        total_discrim_error = 0
        for batch in batches_accumulated:
            with self.accelerator.no_sync(self.netD):

                real_data, labels = batch
                b_size = real_data.size(dim=0)
                accum_labels.append(labels)

                discrim_output = self.netD(real_data, labels)

                real_label = self.create_real_labels(b_size, labels)
                discrim_on_real_error = self.criterion(discrim_output, real_label)
                discrim_on_real_error = discrim_on_real_error / self.accumulation_iterations

                self.accelerator.backward(discrim_on_real_error)

                # Train with all-fake batch
                noise = torch.randn(b_size, self.latent_vector_size, device=self.device, dtype=real_data.dtype)
                with torch.no_grad():
                    # Generate fake image batch with G
                    fake = self.netG(noise, labels)

                # Classify all fake batch with D
                fake_output = self.netD(fake.detach(), labels)

                # Calculate D's loss on the all-fake batch
                fake_label = self.create_fake_labels(b_size)
                discrim_on_fake_error = self.criterion(fake_output, fake_label.reshape_as(fake_output))
                discrim_on_fake_error = discrim_on_fake_error / self.accumulation_iterations
                self.accelerator.backward(discrim_on_fake_error)

                total_discrim_error += discrim_on_real_error + discrim_on_fake_error
        self.optimizerD.step()
        self.optimizerD.zero_grad(set_to_none=True)
        return total_discrim_error.item(), accum_labels

    def generator_step(self, accum_labels):
        self.netD.requires_grad_(requires_grad=False)
        self.optimizerG.zero_grad(set_to_none=True)

        total_generator_error = 0
        for label in accum_labels:
            with self.accelerator.no_sync(self.netG):
                # with self.accelerator.autocast():
                b_size = label.size(dim=0)
                # Train with all-fake batch
                noise = torch.randn(b_size, self.latent_vector_size, device=self.device, requires_grad=True, dtype=self.torch_dtype)
                # Generate fake image batch with G
                fake = self.netG(noise, label)
                self.netD.requires_grad_(requires_grad=False)
                output_update = self.netD(fake, label)
                self.netD.requires_grad_(requires_grad=True)
                real_label = self.create_real_labels(b_size, label)
                generator_error = self.criterion(output_update, real_label) / self.accumulation_iterations

                self.accelerator.backward(generator_error)
                total_generator_error += generator_error

            if not self.orthogonal_value == 0:
                with self.accelerator.autocast():
                    generator_error = self.apply_orthogonal_regularization(self.netG)
                    total_generator_error += generator_error
                self.accelerator.backward(generator_error)
        self.optimizerG.step()
        self.optimizerG.zero_grad(set_to_none=True)
        return total_generator_error.item()

    # Accelerate Models should be done after loading the models
    def accelerate_models(self):
        self.netD, self.netG, self.optimizerD, self.optimizerG = self.accelerator.prepare(
            self.netD, self.netG, self.optimizerD, self.optimizerG
        )

    def apply_orthogonal_regularization(self, model):
        model_orthogonal_loss = 0.0
        for name, param in model.named_parameters():
            if len(param.size()) >= 2 and param.requires_grad and 'conv' in name:
                param = param.view(param.size(dim=0), -1)
                mult_out = torch.mm(torch.t(param), param)
                orthogonal_matrix = torch.mul(mult_out, 1 - torch.eye(mult_out.size(dim=0), device=self.device))
                # Frobenius norm
                model_orthogonal_loss += self.orthogonal_value * torch.norm(orthogonal_matrix, p='fro')
        return model_orthogonal_loss

    # If class_embeddings is provided, then it will ignore labels to generate the images
    def generate_images(self, noise, labels, class_embeddings=None, unnormalize_img=True):
        self.netG.eval() # Set Generater to Eval mode
        with torch.no_grad():
            if self.ema:
                with self.ema.average_parameters():
                    if class_embeddings:
                        forward_with_class_embeddings = getattr(self.netG, "forward_with_class_embeddings", None)
                        if callable(forward_with_class_embeddings):
                            fake = forward_with_class_embeddings(noise, class_embeddings)
                        else:
                            raise ValueError('Given generator model does not support providing class embeddings '
                                                'directly.')
                    else:
                        fake = self.netG(noise, labels)
            else:
                fake = self.netG(noise, labels)

            fake = fake.detach()
            if unnormalize_img:
                fake = unnormalize(fake)
            return fake

    def save(self, model_dir, step_num, compiled=False):
        generator_path = os.path.join(model_dir, os_helper.ModelType.GENERATOR.value + '_step_' + str(step_num) + '.pt')
        discrim_path = os.path.join(model_dir, os_helper.ModelType.DISCRIMINATOR.value + '_step_' +
                                    str(step_num) + '.pt')

        # Save uncompiled model, so you can go from compile to uncompiled
        if compiled:
            netG = self.netG._orig_mod
            netD = self.netD._orig_mod
        else:
            netG = self.netG
            netD = self.netD
        torch.save(netG.state_dict(), generator_path)
        torch.save(netD.state_dict(), discrim_path)
        if self.ema:
            ema_path = os.path.join(model_dir, os_helper.ModelType.EMA.value + '_step_' + str(step_num) + '.pt')
            torch.save(self.ema.state_dict(), ema_path)

    def load(self, generator_path, discrim_path, ema_path=None):
        self.netG.load_state_dict(torch.load(generator_path))
        self.netD.load_state_dict(torch.load(discrim_path))
        if self.ema and ema_path:
            self.ema.load_state_dict(torch.load(ema_path))

    # Writes text file with information of the generator and discriminator instances
    def save_architecture(self, save_dir, data_config):

        image_height, image_width = data_load.get_image_height_and_width(data_config)
        discriminator_stats = summary(self.netD, input_data=[torch.zeros((1, 3, image_height, image_width)),
                                                             torch.zeros(1, dtype=torch.int64)], verbose=0,
                                      device=self.device)
        generator_stats = summary(self.netG, input_data=[torch.zeros(1, self.latent_vector_size),
                                                         torch.zeros(1, dtype=torch.int64)], verbose=0,
                                  device=self.device)

        with open(os.path.join(save_dir, 'architecture.txt'), 'w', encoding='utf-8') as text_file:
            text_file.write('Generator\n\n')
            text_file.write(str(generator_stats))
            text_file.write(str(self.netG))
            text_file.write('\n\nDiscriminator\n\n')
            text_file.write(str(discriminator_stats))
            text_file.write(str(self.netD))


import torch
from datetime import datetime
from pathlib import Path
import os
import argparse
from torch import nn
from torch.utils.data import DataLoader
from torch.distributed import all_gather
from torch.distributed import get_world_size
import torchaudio
from ae_model import CNNAutoencoder
from distributed import init_distributed_mode

from custom_audio_dataset import BirdAudioDataset
from ae_custom_distributed_sampler import CustomDistributedSampler

from torch.utils.tensorboard import SummaryWriter

import scipy.stats as stats

def get_arguments():
    parser = argparse.ArgumentParser(
        description="Pretrain a resnet model with VICReg", add_help=False
    )

    # Optim
    parser.add_argument("--epochs", type=int, default=500, help="Number of epochs")
    parser.add_argument(
        "--batch-size", type=int, default=2048, help="Effective batch size"
    )
    parser.add_argument("--lr", type=float, default=0.0001, help="Learning rate")
    parser.add_argument("--wd", type=float, default=1e-7, help="Weight decay")

    # Checkpoints
    parser.add_argument(
        "--exp-dir",
        type=Path,
        default="./exp",
        help="Path to the experiment folder, where all logs/checkpoints will be stored",
    )
    parser.add_argument(
        "--resume",
        type=bool,
        default=True,
        help="Whether or not to resume from a checkpoint",
    )

    # Running
    parser.add_argument("--num-workers", type=int, default=10)
    parser.add_argument(
        "--device", default="cuda", help="device to use for training / testing"
    )

    # Distributed
    parser.add_argument(
        "--world-size", default=1, type=int, help="number of distributed processes"
    )
    parser.add_argument("--local_rank", default=-1, type=int)
    parser.add_argument(
        "--dist-url", default="env://", help="url used to set up distributed training"
    )

    # Pruning
    parser.add_argument("--prune-anomalies", action='store_true')
    parser.add_argument("--prune-frequency", default=50, type=int)

    # Spectrogram Cropping
    parser.add_argument("--crop-mode", choices=['none', 'second_half', 'third_quarter'], default='third_quarter')

    # Audio Files
    parser.add_argument("--audio-path", type=Path, default="/grand/projects/BirdAudio/Morton_Arboretum/audio/set3/00004879")
    parser.add_argument("--single-file", type=Path)
    parser.add_argument("--audio-file-cap", type=int, default=15)

    # Sampling
    parser.add_argument("--num-samples", type=int, default=22050)
    parser.add_argument("--sample-rate", type=int, default=22050)

    # Logging
    parser.add_argument("--log-frequency", type=int, default=1)

    # Model
    parser.add_argument("--bottleneck-dim", type=int, default=10)

    return parser

def get_all_audio_files(audio_path, audio_file_cap):
    audio_files = []
    for root, dirs, files in os.walk(audio_path, topdown=False):
        for name in files:
            extension = os.path.splitext(name)[1]
            if extension == ".wav":
                audio_file = os.path.join(root, name)
                audio_files.append(audio_file)
                if len(audio_files) >= audio_file_cap:
                    return audio_files
    return audio_files

def train_one_epoch(args, model, data_loader, loss_fn, optimizer, device, epoch, writer, gpu):
    for i, (inputs, _) in enumerate(data_loader, start=epoch * len(data_loader)):
        inputs = inputs.cuda(gpu, non_blocking=True)
        recons, _ = model(inputs)
        loss = loss_fn(recons, inputs)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        writer.add_scalar("Loss", loss.item())
    if (epoch + 1) % args.log_frequency == 0:
        print("Loss: ", loss.item())
    if args.rank == 0:
        state = dict(
            epoch=epoch + 1,
            model=model.state_dict(),
            optimizer=optimizer.state_dict(),
        )
        torch.save(state, args.exp_dir / "model.pth")

def train(
    args,
    model,
    bad,
    data_loader,
    prune_data_loader,
    loss_fn,
    optimizer,
    device,
    epochs,
    start_epoch,
    writer,
    gpu,
):
    start_time = datetime.now()
    for epoch in range(start_epoch, epochs):
        print(f"Epoch:  {epoch + 1}/{epochs}, Time: {datetime.now() - start_time}")
        if args.prune_anomalies and (epoch + 1) % args.prune_frequency == 0:
            print("Pruning anomalies!")
            normal_indices = prune_anomalies(model, bad, prune_data_loader, loss_fn, gpu)
            data_loader, prune_data_loader = get_data_loaders(args, bad, normal_indices)
        train_one_epoch(
            args, model, data_loader, loss_fn, optimizer, device, epoch, writer, gpu
        )
    print("Training is done.")
    if args.rank == 0:
        torch.save(model.state_dict(), args.exp_dir / "conv_autoencoder.pth")

def prune_anomalies(model, bad, prune_data_loader, loss_fn, gpu):
    indices = []
    losses = []

    with torch.no_grad():
        for i, (inputs, _) in enumerate(prune_data_loader):
            inputs = inputs.cuda(gpu, non_blocking=True)
            recons, _ = model(inputs)
            loss = loss_fn(recons, inputs)
            indices.append(i)
            losses.append(loss)

    indices = torch.tensor(indices, dtype=torch.int32).cuda(gpu, non_blocking=True)
    losses = torch.tensor(losses, dtype=torch.float32).cuda(gpu, non_blocking=True)

    all_indices = [
        torch.zeros(len(prune_data_loader), dtype=torch.int32).cuda(gpu, non_blocking=True) for _ in range(get_world_size())
    ]
    all_losses = [
        torch.zeros(len(prune_data_loader), dtype=torch.float32).cuda(gpu, non_blocking=True) for _ in range(get_world_size())
    ]

    all_gather(all_indices, indices)
    all_gather(all_losses, losses)

    all_indices = torch.cat(all_indices).detach().cpu().numpy()
    all_losses = torch.cat(all_losses).detach().cpu().numpy()

    all_z_scores = stats.zscore(all_losses)

    normal_indices = all_indices[all_z_scores < 3]

    print(f"Pruned {len(all_indices) - len(normal_indices)} samples...")

    return normal_indices

def main(args):
    torch.backends.cudnn.benchmark = True
    init_distributed_mode(args)
    print(args)
    gpu = torch.device(args.device)

    if args.rank == 0:
        args.exp_dir.mkdir(parents=True, exist_ok=True)

    mel_spectrogram = torchaudio.transforms.MelSpectrogram(
        sample_rate=args.sample_rate,
        n_fft=1024,
        hop_length=512,
        n_mels=((256 if args.crop_mode == "third_quarter" else (128 if args.crop_mode == "second_half" else 64)))
    )

    if args.single_file is not None:
        audio_files = [args.single_file]
        print(f"Loading single audio file: ", args. single_file)

    else:
        audio_files = get_all_audio_files(args.audio_path, args.audio_file_cap)
        print(f"Loading {len(audio_files)} audio file(s):")
        for audio_file in audio_files:
            print(audio_file)

    bad = BirdAudioDataset(
        audio_files,
        mel_spectrogram,
        args.sample_rate,
        args.num_samples,
        gpu,
        crop_mode=args.crop_mode
    )

    train_data_loader, prune_data_loader = get_data_loaders(args, bad, range(len(bad)))

    model = CNNAutoencoder(bottleneck_dim=args.bottleneck_dim).cuda(gpu)
    if torch.cuda.device_count() > 1:
        print("Using", torch.cuda.device_count(), "GPUs")
    model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])

    loss_fn = nn.MSELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)

    if (args.exp_dir / "model.pth").is_file() and args.resume:
        if args.rank == 0:
            print("resuming from checkpoint")
        ckpt = torch.load(args.exp_dir / "model.pth", map_location="cpu")
        start_epoch = ckpt["epoch"]
        model.load_state_dict(ckpt["model"])
        optimizer.load_state_dict(ckpt["optimizer"])
    else:
        start_epoch = 0

    log_dir = f"theta_run_birds/lr{args.lr}_wd{args.wd}_bs{args.batch_size}_epochs{args.epochs}"
    writer = SummaryWriter(log_dir)
    print(f"Tensorboard logging at {log_dir}")

    train(
        args,
        model,
        bad,
        train_data_loader,
        prune_data_loader,
        loss_fn,
        optimizer,
        args.device,
        args.epochs,
        start_epoch,
        writer,
        gpu,
    )

def get_data_loaders(args, bad, normal_indices):
    assert args.batch_size % args.world_size == 0
    per_device_batch_size = args.batch_size // args.world_size

    sampler = CustomDistributedSampler(bad, normal_indices)
    print("Number of samples: ", sampler.total_size)

    train_data_loader = DataLoader(
        bad,
        batch_size=per_device_batch_size,
        num_workers=args.num_workers,
        pin_memory=False,
        sampler=sampler,
    )
    prune_data_loader = DataLoader(
        bad,
        batch_size=1,
        num_workers=args.num_workers,
        pin_memory=False,
        sampler=sampler
    )

    return train_data_loader, prune_data_loader

if __name__ == "__main__":
    parser = get_arguments()
    args = parser.parse_args()
    main(args)


import pandas as pd
import numpy as np
if __name__ == '__main__':
    df = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'],
                         'data1': range(6)})
    #If a column in a DataFrame has k distinct values, you would derive a matrix or
    # Data‐ Frame with k columns containing all 1s and 0s.
    # pandas has a get_dummies function for doing this,
    dummydf = pd.get_dummies(df['key'])
    print(dummydf)

    dummies = pd.get_dummies(df['key'], prefix='key')
    df_with_dummy = df[['data1']].join(dummies)
    print(df_with_dummy)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 22 00:40:40 2017

@author: shivamchaturvedi
"""

'''
====================================================
Case Study 2 - Word Frequency Distribution in Hamlet
====================================================
'''

#In this case study, we will find and plot the distribution of word frequencies for each translation of Hamlet.
#Perhaps the distribution of word frequencies of Hamlet depends on the translation --- let's find out!

#For these exercises, functions count_words_fast, read_book, and word_stats are already defined as in the Case 2 Videos (Videos 3.2.x).

#Define functions
#------------------

from collections import Counter

def count_words_fast(text):
    """count the number of times each word occurs in text (str).
    Return dictionary where keys are unique words and values are
    word counts. skip punctuations"""
    text = text.lower() #lowercase for the counting letters so the function can cont the same words whether it's capatilised or not
    skips = [".", ",", ";", ":", "'", '"'] #skipping all the punctuations to not be counted with the words that come bfore them
    for ch in skips:
        text = text.replace(ch,"")
    word_counts = Counter(text.split(" "))
    return word_counts

#------------------
def read_book(title_path):
    """Read a book and return it as a string"""
    with open(title_path, "r") as current_file:
        text = current_file.read()
        text = text.replace("\n","").replace("\r","")
    return text

#------------------
def word_stats(word_counts):
   """return the number of unique words and word frequencies"""
   num_unique = len(word_counts) #calculate the number of unique words in the text
   counts = word_counts.values() #calculate the frequency of each word in the text
   return(num_unique,counts)

#----------------------------------------------------------------------------------------------------------------

# Exercise 1
#-----------

#TODO: Write a function word_count_distribution(text) that takes a book string and returns a dictionary with items
#corresponding to the count of times a collection of words appears in the translation, and values corresponding to
#the number of number of words that appear with that frequency.

#TODO: First use count_words_fast(text) to create a dictionary called word_counts with unique words in the dictionary
#as keys and their frequency in the book as values.

#TODO: Next, create and return a new dictionary count_distribution with unique values from word_counts as keys and their
#frequency as values. For example, 'you are what you eat' contains three words that occur once and one word that occurs twice,
#so word_count_distribution('you are what you eat') should return a dictionary {1:3, 2:1}.

def word_count_distribution(text):
    word_counts = count_words_fast(text)
    count_distribution = dict(Counter(word_counts.values()))
    return count_distribution

#TODO: 'Romeo and Juliet' is preloaded as text. Call word_count_distribution(text), and save the result as distribution.

distribution = word_count_distribution(text)

print(distribution)
#------------------------------------------------------------------------------

# Exercise 2
#-----------

#TODO: Create a function more_frequent(distribution) that takes a word frequency dictionary (like that made in Exercise 1)
# and outputs a dictionary with the same keys as those in distribution
# (the number of times a group of words appears in the text), and values corresponding to the fraction of words that occur
# with more frequency than that key.

import numpy as np

def more_frequent(distribution):
    counts = list(distribution.keys())
    frequency_of_counts = list(distribution.values())
    cumulative_frequencies = np.cumsum(frequency_of_counts)
    more_frequent = 1 - cumulative_frequencies / cumulative_frequencies[-1] #To obtain the fraction of words more frequent than this,
    #divide this cumulative sum by its maximum, and subtract this value from 1.
    return dict(zip(counts, more_frequent))

#TODO: Call more_frequent(distribution).

more_frequent(distribution)

#------------------------------------------------------------------------------

# Exercise 3
#-----------

#Edit the code used to read though each of the books in our library, and store
#the word frequency distribution for each translation of William Shakespeare's
#"Hamlet" as a Pandas dataframe hamlets with columns named "language" and
#"distribution". word_count_distribution is preloaded from Exercise 1.
#How many translations are there? Which languages are they translated into?

import pandas as pd

hamlets = pd.DataFrame(columns=('language','distribution'))

book_dir = "Books"
title_num = 1

for language in book_titles:
    for author in book_titles[language]:
        for title in book_titles[language][author]:
            if title == "Hamlet":
                inputfile = data_filepath+"Books/"+language+"/"+author+"/"+title+".txt"
                text = read_book(inputfile)
                distribution = word_count_distribution(text)
                hamlets.loc[title_num] = language, distribution
                title_num += 1

#There are three translations: English, German, and Portuguese.

#------------------------------------------------------------------------------

# Exercise 4
#-----------

#Plot the word frequency distributions of each translations on a single log-log plot.
#Note that we have already done most of the work for you. Do the distributions of each translation differ?

import matplotlib.pyplot as plt

colors = ["crimson", "forestgreen", "blueviolet"]
handles, hamlet_languages = [], []
for index in range(hamlets.shape[0]):
    language, distribution = hamlets.language[index+1], hamlets.distribution[index+1]
    dist = more_frequent(distribution)
    plot, = plt.loglog(sorted(list(dist.keys())),sorted(list(dist.values()),
        reverse = True), color = colors[index], linewidth = 2)
    handles.append(plot)
    hamlet_languages.append(language)
plt.title("Word Frequencies in Hamlet Translations")
xlim    = [0, 2e3]
xlabel  = "Frequency of Word $W$"
ylabel  = "Fraction of Words\nWith Greater Frequency than $W$"
plt.xlim(xlim); plt.xlabel(xlabel); plt.ylabel(ylabel)
plt.legend(handles, hamlet_languages, loc = "upper right", numpoints = 1)
plt.show()

#The distributions differ somewhat, but their basic shape is the same.
#By the way, distributions that look like a straight line like these are called
#'scale-free,' because the line looks the same no matter where on the x-axis you look!


# Definition for singly-linked list.
# class ListNode:
#     def init(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
    def middleNode(self, head: Optional[ListNode]) -> Optional[ListNode]:
        ans = ListNode(0)
        ansL = ans
        output = []
        while(head):
            output.append(head.val)
            head = head.next
        x = len(output)//2
        output = output[x:]
        for i in output:
            ansL.next = ListNode(i)
            ansL = ansL.next
        return ans.next


import matplotlib.pyplot as plt  # Importing matplotlib for visualization
import pandas as pd  # Importing pandas for data manipulation and analysis
import numpy as np  # Importing numpy for numerical operations
import xlrd  # Importing xlrd to read data from Excel files
import xlwt  # Importing xlwt to write data to Excel files
import joblib  # Importing joblib to save and load models
import pyautogui  # Importing pyautogui for GUI automation
import math  # Importing math for mathematical operations
import pyperclip  # Importing pyperclip for clipboard operations
import cv2  # Importing cv2 for image processing
import csv  # Importing csv for reading and writing CSV files
import random  # Importing random for random number generation
import warnings  # Importing warnings to ignore warnings
import os  # Importing os for operating system-related operations
import time  # Importing time to measure and manipulate time
from eppy.modeleditor import IDF  # Importing IDF from eppy to edit and manipulate EnergyPlus models
from eppy.results import readhtml  # Importing readhtml from eppy.results to read EnergyPlus HTML output files
from ml_utils import *  # Importing ml_utils to use machine learning utilities

# Ignore warnings
warnings.filterwarnings("ignore")

# Set working directory
os.chdir('E:/surrogate model')

# Set EnergyPlus IDD file
iddfile = "Energy+.idd"

# Define the simulated sleep times in hours
Simulate_sleep_times = [0, 3, 6, 10, 14, 18, 22, 26, 28, 32, 36]

# Define the different envelope paths
Envelope_Paths = ['reference', 'moderate', 'radical']

# Define the different weather scenarios
Weather_Scenarios = ['ssp126', 'ssp245', 'ssp585']

# Define the years to observe
year_observes = [2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043,
                 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067,
                 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090, 2091,
                 2092, 2093, 2094, 2095, 2096, 2097, 2098, 2099, 2100]

# List of variables used for heating model training
Model_training_variable_heating = ['roof_u-factor', 'wall_u-factor', 'window_u-factor', 'day', 'hour_cluster_min', 'hour_cluster_mean', 'month_cluster_mean',
                                  'month_cluster_min', 'stories_cluster_mean', 'stories_cluster_min', 'tas', 'tdew', 'rh', 'pressure', 'Horizontal Infrared Radiation Intensity',
                                  'Global Horizontal Radiation', 'Direct Normal Radiation', 'Diffuse Horizontal Radiation', 'wind', 'wind speed', 'Total Sky Cover', 'Opaque Sky Cover',
                                  'lon', 'lat', 'high', 'location_cluster_coldB', 'location_cluster_severe coldA', 'location_cluster_severe coldB',
                                  'location_cluster_severe coldC', 'location_cluster_warm', 'location_cluster_winter cold', 'location_cluster_winter warm']

# List of variables used for cooling model training
Model_training_variable_cooling = ['roof_u-factor', 'wall_u-factor', 'window_u-factor', 'day', 'hour_cluster_min', 'hour_cluster_mean',
                                   'month_cluster_mean', 'month_cluster_min', 'stories_cluster_min', 'tas', 'tdew', 'rh', 'pressure', 'Horizontal Infrared Radiation Intensity',
                                   'Global Horizontal Radiation', 'Direct Normal Radiation', 'Diffuse Horizontal Radiation', 'wind',
                                   'wind speed', 'Total Sky Cover', 'Opaque Sky Cover', 'lon', 'lat', 'high', 'location_cluster_coldB', 'location_cluster_severe coldA', 'location_cluster_severe coldB',
                                   'location_cluster_severe coldC', 'location_cluster_warm', 'location_cluster_winter cold', 'location_cluster_winter warm']

# List of all variables used in the models
variables_name = ['roof_u-factor', 'wall_u-factor', 'window_u-factor', 'stories', 'month', 'day', 'hour', 'tas', 'tdew', 'rh', 'pressure', 'Horizontal Infrared Radiation Intensity',
                  'Global Horizontal Radiation', 'Direct Normal Radiation', 'Diffuse Horizontal Radiation', 'wind', 'wind speed', 'Total Sky Cover', 'Opaque Sky Cover',
                  'location', 'lon', 'lat', 'high', 'heatingenergy', 'coolingenergy']

# Read in the CSV file containing city information into a pandas DataFrame
city_information_table = pd.read_csv('Information of 1667 cities.csv')

# Define a function to read an Excel file given its path and sheet index
def read_excel(path, index):
    # Open the Excel file
    book = xlrd.open_workbook(path)
    # Select the sheet by index
    sheet = book.sheet_by_index(index)
    # Get the number of rows in the sheet
    nrows = sheet.nrows
    # Get the number of columns in the sheet
    ncols = sheet.ncols
    # Return the sheet object, number of rows, and number of columns
    return sheet, nrows, ncols

# Define a function to copy a file from one path to another
def File_SaveAs(f1, f2):
    # Open the source file in read-binary mode
    with open(f1, 'rb') as fp1:
        # Read the contents of the file into a byte string
        b1 = fp1.read()
    # Open the destination file in write-binary mode
    with open(f2, 'wb') as fp2:
        # Write the contents of the source file to the destination file
        fp2.write(b1)

# Process the upper and lower limits of sampling variables to generate a specified number of random samples
def Sample_input_data_process (sample_number, sample_limit):
    # Obtaining sample range
    coefficient_lower = np.zeros((sample_number, 2))  # initialize coefficient lower array
    coefficient_upper = np.zeros((sample_number, 2))  # initialize coefficient upper array
    for i in range(sample_number):
        coefficient_lower[i, 0] = 1 - i / sample_number  # calculate lower coefficient values
        coefficient_lower[i, 1] = i / sample_number  # calculate lower coefficient values
    for i in range(sample_number):
        coefficient_upper[i, 0] = 1-(i+1) / sample_number  # calculate upper coefficient values
        coefficient_upper[i, 1] = (i+1) / sample_number  # calculate upper coefficient values
    sample_limit_lower = coefficient_lower @ sample_limit.T  # calculate lower limit for sample range
    sample_limit_upper = coefficient_upper @ sample_limit.T  # calculate upper limit for sample range
    sample_range = np.dstack((sample_limit_lower.T, sample_limit_upper.T))  # stack lower and upper limits

    # Random sampling in the sample range
    sample_variable_number = sample_range.shape[0]  # number of variables to sample
    sample_layer_number  = sample_range.shape[1]  # number of layers to sample
    coefficient_random = np.zeros((sample_variable_number,sample_layer_number, 2))  # initialize coefficient random array
    sample_matrix = np.zeros((sample_layer_number, sample_variable_number))  # initialize sample matrix

    for m in range(sample_variable_number):
        for i in range(sample_layer_number):
            y = random.random()  # generate a random number
            coefficient_random[m, i, 0] = 1 - y  # calculate lower coefficient values
            coefficient_random[m, i, 1] = y  # calculate upper coefficient values
    temp_arr = sample_range * coefficient_random  # generate temporary array

    for j in range(sample_variable_number):
        temp_random = temp_arr[j, :, 0] + temp_arr[j, :, 1]  # calculate random values within the range
        sample_matrix[:,j] = temp_random  # add random values to the sample matrix

    for n in range(sample_matrix.shape[1]):
        np.random.shuffle(sample_matrix[:, n])  # disrupt the order of the values in each column
    return sample_matrix

# For all the processes related to the sample, only one parameter is left, that is, sample_number
def Sample(sample_number):
    # Given sampled variables and variable limits
    sample_limit = np.array([[0.08, 0.08, 0.25, 0.5, 0.5, 0.5, 0.5],
                          [1.98, 1.5, 3.75, 10.5, 270.5, 8760.5, 1752.5]]).T
    sample_name = ["roof_u-factor", "wall_u-factor", "window_u-factor", "stories", 'location', 'extract_start', 'extract_interval']
    sample_matrix = Sample_input_data_process(sample_number, sample_limit)
    sample_output_data = pd.DataFrame(sample_matrix, columns = sample_name)

    # Keep the decimals as desired for each variable
    sample_output_data['roof_u-factor'] = sample_output_data['roof_u-factor'].round(3)
    sample_output_data['wall_u-factor'] = sample_output_data['wall_u-factor'].round(3)
    sample_output_data['window_u-factor'] = sample_output_data['window_u-factor'].round(3)
    sample_output_data['stories'] = sample_output_data['stories'].round(0)
    sample_output_data['location'] = sample_output_data['location'].round(0)
    sample_output_data['extract_start'] = sample_output_data['extract_start'].round(0)
    sample_output_data['extract_interval'] = sample_output_data['extract_interval'].round(0)
    #sample_output_data.to_excel('./sample_output.xls')

# Process the files used for simulation according to the parameters specified in the sample
def Simulate_input_data_process(sample_order):
    # Read the sample set generated by sampling and process it sample by sample
    path_sample_output = './sample_output.xls'
    sheet_sample_output, nrows_sample_output, ncols_sample_output = read_excel(path_sample_output, 0)

    # Select the weather file specified by the sample, and change it to a unified simulation file
    simulate_input_weather_station = int(sheet_sample_output.cell_value(sample_order, 5))
    path_epwfile = './weather_energyplus/{simulate_input_weather_station}_weather_site.epw'.format(simulate_input_weather_station = simulate_input_weather_station)
    path_simulate_input_weather = './overall/new.epw'
    File_SaveAs(path_epwfile, path_simulate_input_weather)

    # Select the building file specified by the sample, and change it to a unified simulation file
    path_roof_ufactor = 'roof_u-factor.xls'
    path_wall_ufactor = 'wall_u-factor.xls'
    path_window_ufactor = 'window_u-factor.xls'
    sheet_roof_ufactor, nrows_roof_ufactor, ncols_roof_ufactor = read_excel(path_roof_ufactor, 0)
    sheet_wall_ufactor, nrows_wall_ufactor, ncols_wall_ufactor = read_excel(path_wall_ufactor, 0)
    sheet_window_ufactor, nrows_window_ufactor, ncols_window_ufactor = read_excel(path_window_ufactor, 0)

    sample_envelopes = []
    for col in range(1, 4):
        sample_envelopes.append(sheet_sample_output.cell_value(sample_order, col))
    simulate_input_building_story = ('%.0f' % float(sheet_sample_output.cell_value(sample_order, 4)))
    IDF.setiddname(iddfile)
    simulate_input_building = IDF('./prototype/simpletrain{simulate_input_building_story}.idf'.format(simulate_input_building_story = simulate_input_building_story))

    # Modify the heat transfer coefficients of the three main constructions of the building envelope
    material_roof = simulate_input_building.idfobjects['MATERIAL'][6]
    for row_roof_ufactor in range(1, nrows_roof_ufactor):
        if ('%.3f' %float(sheet_roof_ufactor.cell_value(row_roof_ufactor, 1))) == ('%.3f' %float(sample_envelopes[0])):
            ufactor_roof = sheet_roof_ufactor.cell_value(row_roof_ufactor, 0)
    material_roof.Conductivity = ufactor_roof

    material_wall = simulate_input_building.idfobjects['MATERIAL'][3]
    for row_wall_ufactor in range(1, nrows_wall_ufactor):
        if ('%.3f' %float(sheet_wall_ufactor.cell_value(row_wall_ufactor, 1))) == ('%.3f' %float(sample_envelopes[1])):
            ufactor_wall = sheet_wall_ufactor.cell_value(row_wall_ufactor, 0)
    material_wall.Conductivity = ufactor_wall

    material_window = simulate_input_building.idfobjects['WindowMATERIAL:Glazing'][0]
    for row_window_ufactor in range(1, nrows_window_ufactor):
        if ('%.3f' %float(sheet_window_ufactor.cell_value(row_window_ufactor, 1))) == ('%.3f' %float(sample_envelopes[2])):
            ufactor_window = sheet_window_ufactor.cell_value(row_window_ufactor, 0)
    material_window.Conductivity = ufactor_window
    material_window.Solar_Transmittance_at_Normal_Incidence = 0.5

    # Add the ddy file for each locale to the building file
    ddyidf = IDF('./prototype/weather_energyplus/{simulate_input_weather_station}_weather_site.ddy'.format(simulate_input_weather_station=simulate_input_weather_station))
    designdays = ddyidf.idfobjects["SizingPeriod:DesignDay".upper()]
    designday_cooling = designdays[6]
    designday_heating = designdays[0]
    # copy a design day into your idf file
    simulate_input_building.copyidfobject(designday_cooling)
    simulate_input_building.copyidfobject(designday_heating)
    simulate_input_building.saveas('prototype/new/new.idf')

    return simulate_input_building, simulate_input_building_story

# Run the simulation software and get the energy consumption intensity file of the simulation output
def Simulate(sample_order):
    # Process the input data for simulation
    simulate_input_building, simulate_input_building_story = Simulate_input_data_process(sample_order)

    # Find and click the "Simulate" button, then sleep for a certain amount of time
    simulate_location_begin = pyautogui.locateOnScreen(image = 'prototype/new/simulate.png', confidence = 0.56)
    simulate_location_begin_x, simulate_location_begin_y = pyautogui.center(simulate_location_begin)
    pyautogui.moveTo(simulate_location_begin_x, simulate_location_begin_y, duration = 1)
    pyautogui.click()
    time.sleep(Simulate_sleep_times[int(simulate_input_building_story)])

    # Find and click the "OK" button to end the simulation
    simulate_location_end = pyautogui.locateOnScreen(image = 'prototype/new/ok.png', confidence = 0.7)
    while simulate_location_end is None:
        simulate_location_end = pyautogui.locateOnScreen(image = 'prototype/new/ok.png', confidence = 0.7)
    simulate_location_end_x, simulate_location_end_y = pyautogui.center(simulate_location_end)
    pyautogui.moveTo(simulate_location_end_x, simulate_location_end_y, duration = 1)
    pyautogui.click()

    # Read the simulation output data and save it to a csv file
    dtf_simulate_output = pd.read_csv('prototype/new/new.csv')
    dtf_simulate_output.to_csv('./simulate_output_intensity/coldA_{sample_order}.csv'.format(sample_order = sample_order))

# The collation of data into a standardized data format that can be used for training models
def Train_input_data_produce(sample_order):
    # Read the sampling results of the Latin hypercube
    path_sample_output = './sample_output.xls'
    sheet_sample_output, nrows_sample_output, ncols_sample_output = read_excel(path_sample_output, 0)

    # Read all seven variables sampled line by line
    train_samples = []
    for row in range(1, sample_order):
        print(row)
        sample_envelopes = []  # Create an empty list to store the envelope information of the current sample
        sample_story = int(sheet_sample_output.cell_value(row,
                                                          4))  # Get the number of stories for the current sample from the Excel sheet
        sample_order = int(
            sheet_sample_output.cell_value(row, 0)) + 1  # Get the order of the current sample from the Excel sheet
        sample_station = int(sheet_sample_output.cell_value(row,
                                                            5))  # Get the weather station number for the current sample from the Excel sheet
        sample_simulate_output_start = int(sheet_sample_output.cell_value(row,
                                                                          6))  # Get the starting time point for the simulation output for the current sample from the Excel sheet
        sample_simulate_output_interval = int(sheet_sample_output.cell_value(row,
                                                                             7))  # Get the time interval for the simulation output for the current sample from the Excel sheet

        # Store the envelope information of the current sample in the sample_envelopes list
        sample_envelopes.append(sheet_sample_output.cell_value(row, 1))
        sample_envelopes.append(sheet_sample_output.cell_value(row, 2))
        sample_envelopes.append(sheet_sample_output.cell_value(row, 3))
        sample_envelopes.append(sample_story)

        path_train_input_weather = './train_input_weather/{sample_station}_weather_site_hours.csv'.format(
            sample_station=int(sample_station))
        path_simulate_output_intensity = './simulate_output_intensity/coldA_{sample_order}.csv'.format(
            sample_order=sample_order)
        dtf_train_input_weather = pd.read_csv(path_train_input_weather)
        dtf_train_input_intensity = pd.read_csv(path_simulate_output_intensity)
        dtf_train_input_weather = dtf_train_input_weather.drop('Unnamed: 0', axis=1)
        dtf_train_input_intensity = dtf_train_input_intensity.drop('Unnamed: 0', axis=1)
        dtf_train_input_intensity = dtf_train_input_intensity.drop('Date/Time', axis=1)
        dtf_train_input_intensity = pd.DataFrame(dtf_train_input_intensity.values.T,
                                                 index=dtf_train_input_intensity.columns,
                                                 columns=dtf_train_input_intensity.index)

        # Calculate the heating and cooling loads for each hour of the current sample and store them in two separate DataFrames
        dtf_train_input_heating = dtf_train_input_intensity[:sample_story * 4].apply(lambda x: x.sum(),
                                                                                axis=0) / sample_story / 783.65
        dtf_train_input_cooling = dtf_train_input_intensity[sample_story * 4:].apply(lambda x: x.sum(),
                                                                                axis=0) / sample_story / 783.65

        for j in range(5):  # Iterate through the five output time points for each sample
            sample_simulate_output_point = sample_simulate_output_start + j * sample_simulate_output_interval  # Calculate the current output time point
            if sample_simulate_output_point > 8760:  # If the current output time point is beyond one year, subtract 8760 to keep it within the range of one year
                sample_simulate_output_point = sample_simulate_output_point - 8760
            train_sample = pd.concat(
                [dtf_train_input_weather[sample_simulate_output_point - 1: sample_simulate_output_point],
                 dtf_train_input_heating[sample_simulate_output_point - 1: sample_simulate_output_point],
                 dtf_train_input_cooling[sample_simulate_output_point - 1: sample_simulate_output_point]], axis=1,
                join='inner')
            train_sample = train_sample.reset_index()
            train_sample = train_sample.drop('index', axis=1)
            train_sample = pd.concat([pd.DataFrame(pd.DataFrame(sample_envelopes).values.T), train_sample], axis=1,
                                     join='inner')
            train_samples.extend(np.array(train_sample).tolist())

    # Combine all the training data samples into a pandas dataframe and save it to a csv file
    train_samples = pd.DataFrame(train_samples)
    train_samples.columns = variables_name
    train_samples.to_csv('./train_input.csv')

# Analysis of the relationship between independent variables and target variables in training samples
def Train_input_data_sanalyze(dtf):
    # Univariate analysis
    freqdist_plot(dtf, "Y", box_logscale=True, figsize=(10,5)) # Plot the frequency distribution of the target variable 'Y' on a logarithmic scale
    freqdist_plot(dtf, "stories", figsize=(5,3)) # Plot the frequency distribution of the feature 'stories'

    # Binary distribution analysis
    bivariate_plot(dtf, x="roof_u-factor", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'roof_u-factor' and 'Y'
    bivariate_plot(dtf, x="wall_u-factor", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'wall_u-factor' and 'Y'
    bivariate_plot(dtf, x="window_u-factor", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'window_u-factor' and 'Y'
    bivariate_plot(dtf, x="stories", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'stories' and 'Y'
    bivariate_plot(dtf, x="tas", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'tas' and 'Y'
    bivariate_plot(dtf, x="tdew", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'tdew' and 'Y'
    bivariate_plot(dtf, x="rh", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'rh' and 'Y'
    bivariate_plot(dtf, x="pressure", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'pressure' and 'Y'
    bivariate_plot(dtf, x='Horizontal Infrared Radiation Intensity', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'Horizontal Infrared Radiation Intensity' and 'Y'
    bivariate_plot(dtf, x='Global Horizontal Radiation', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'Global Horizontal Radiation' and 'Y'
    bivariate_plot(dtf, x='Direct Normal Radiation', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'Direct Normal Radiation' and 'Y'
    bivariate_plot(dtf, x='Diffuse Horizontal Radiation', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'Diffuse Horizontal Radiation' and 'Y'
    bivariate_plot(dtf, x='wind', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'wind' and 'Y'
    bivariate_plot(dtf, x='wind speed', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'wind speed' and 'Y'
    bivariate_plot(dtf, x='Total Sky Cover', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'Total Sky Cover' and 'Y'
    bivariate_plot(dtf, x='Opaque Sky Cover', y="Y", figsize=(15, 5)) # Plot the bivariate distribution of 'Opaque Sky Cover' and 'Y'
    bivariate_plot(dtf, x="month", y="Y", figsize=(15,5)) # Plot the bivariate distribution of 'month' and 'Y'
    bivariate_plot(dtf, x="hour", y="Y", figsize=(15, 5))  # plot Y against hour
    bivariate_plot(dtf, x="day", y="Y", figsize=(15, 5))  # plot Y against day
    bivariate_plot(dtf, x="location", y="Y", figsize=(15, 5))  # plot Y against location
    bivariate_plot(dtf, x="lon", y="Y", figsize=(15, 5))  # plot Y against longitude
    bivariate_plot(dtf, x="lat", y="Y", figsize=(15, 5))  # plot Y against latitude
    bivariate_plot(dtf, x="high", y="Y", figsize=(15, 5))  # plot Y against high

    # Relevance and significance analysis
    corr = corr_matrix(dtf, method="pearson", negative=False, annotation=True,
                       figsize=(15, 7))  # compute correlation matrix
    dic_feat_sel = features_selection(dtf, y="Y", task="regression", top=10,
                                      figsize=(10, 5))  # select top 10 features using regression task
    model = ensemble.RandomForestRegressor(n_estimators=100, criterion="mse",
                                           random_state=0)  # create a random forest regressor model
    feat_imp = features_importance(X=dtf.drop("Y", axis=1).values, y=dtf["Y"].values,
                                   X_names=dtf.drop("Y", axis=1).columns.tolist(), model=model, task="regression",
                                   figsize=(15, 5))  # compute feature importance using the random forest model

# The preprocessing of the training input data returns the processed data and scaling parameters
def Train_input_data_process():
    # Reading training input data
    dtf_train_input = pd.read_csv('./train.csv')
    for index,row in dtf_train_input.iterrows():
        if row['heatingenergy']!=0 and row['coolingenergy']!=0:
            dtf_train_input = dtf_train_input.drop(index)

    # Different ways to merge data for different energy consumption conditions
    # hours
    hour_clusters_heating = {"max": [], "mean": [20, 21, 22, 23, 24, 11, 12], "min": [13, 14, 15, 16, 17, 18, 19]}
    hour_clusters_cooling = {"max": [14, 15, 16, 17, 18, 19, 20, 21], "mean": [11, 12, 13, 22, 23, 24], "min": []}
    # months
    month_clusters_heating = {"max": [1, 12], "mean": [2, 11], "min": []}
    month_clusters_cooling = {"max": [6, 7, 8], "mean": [5, 9], "min": []}
    # city_building_stories
    city_building_stories_clusters_heating = {"max": [1], "mean": [2, 3], "min": [4, 5, 6, 7, 8, 9, 10]}
    city_building_stories_clusters_cooling = {"max": [], "min": [1]}
    # location
    location_clusters = {
        "coldA": [29, 32, 33, 35, 36, 39, 41, 73, 75, 76, 135, 136, 137, 138, 141, 143, 162, 174, 175,
                  176, 184, 191, 196, 207, 210, 211, 212, 216, 218, 219, 221, 222, 225, 226, 227, 229,
                  231, 233, 235, 237, 240, 241, 242, 244, 250, 251, 252, 253, 254],
        "coldB": [3, 9, 10, 59, 60, 62, 64, 77, 78, 79, 121, 122, 125, 206, 208, 209, 213, 220, 224,
                  228, 230, 243, 246, 247],
        "severe coldA": [38, 89, 93, 95, 99, 146, 150, 157, 167, 171, 178, 180, 183, 186, 187],
        "severe coldB": [30, 80, 84, 85, 86, 87, 88, 90, 91, 92, 94, 96, 97, 98, 100, 115, 145, 152, 154,
                         156, 161, 169, 172, 173, 177, 179, 182, 185, 188, 190, 194, 198, 232, 234, 236],
        "severe coldC": [31, 34, 40, 74, 101, 112, 113, 114, 116, 117, 118, 119, 134, 139, 140, 142, 144,
                         147, 148, 149, 151, 153, 155, 158, 159, 160, 163, 164, 165, 166, 168, 170, 181,
                         189, 193, 202, 223, 238, 239, 245, 248, 249, 256],
        "winter cold": [],
        "winter warm": [13, 14, 18, 20, 21, 22, 24, 25, 27, 28, 42, 43, 45, 46, 47, 48, 49, 50, 51, 81,
                        82, 83, 261, 265],
        "warm": [52, 53, 56, 57, 195, 204, 255, 257, 258, 259, 260, 262, 263, 264]}

    # Determine the scaling rule
    # RobustScaler transforms features by subtracting the median and then dividing by the interquartile range(75 % to 25 % value).
    RobustScalerX_heating = preprocessing.RobustScaler(quantile_range=(25.0, 75.0))
    RobustScalerY_heating = preprocessing.RobustScaler(quantile_range=(25.0, 75.0))
    RobustScalerX_cooling = preprocessing.RobustScaler(quantile_range=(25.0, 75.0))
    RobustScalerY_cooling = preprocessing.RobustScaler(quantile_range=(25.0, 75.0))

    # disposal data for heating
    dtf_train_input_heating = dtf_train_input.rename(columns={"heatingenergy": "Y"})
    dtf_train_input_heating = dtf_train_input_heating.drop('coolingenergy', axis=1)

    # data clusters of heating
    dtf_train_input_heating = add_feature_clusters(dtf_train_input_heating, "hour",
                                                   dic_clusters_mapping=hour_clusters_heating, dropx=False)
    dtf_train_input_heating = add_dummies(dtf_train_input_heating, x="hour_cluster", dropx=True)
    dtf_train_input_heating = add_feature_clusters(dtf_train_input_heating, "month",
                                                   dic_clusters_mapping=month_clusters_heating, dropx=False)
    dtf_train_input_heating = add_dummies(dtf_train_input_heating, x="month_cluster", dropx=True)
    dtf_train_input_heating = add_feature_clusters(dtf_train_input_heating, "stories",
                                                   dic_clusters_mapping=city_building_stories_clusters_heating,
                                                   dropx=False)
    dtf_train_input_heating = add_dummies(dtf_train_input_heating, x="stories_cluster", dropx=True)
    dtf_train_input_heating = add_feature_clusters(dtf_train_input_heating, "location",
                                                   dic_clusters_mapping=location_clusters, dropx=False)
    dtf_train_input_heating = add_dummies(dtf_train_input_heating, x="location_cluster", dropx=True)

    # drop data for heating
    dtf_train_input_heating = dtf_train_input_heating.drop('hour', axis=1)
    dtf_train_input_heating = dtf_train_input_heating.drop('month', axis=1)
    dtf_train_input_heating = dtf_train_input_heating.drop('stories', axis=1)
    dtf_train_input_heating = dtf_train_input_heating.drop('location', axis=1)

    # Calculate the scaling parameters of heating
    dtf_train_input_heating, scalerX_heating, scalerY_heating = scaling(dtf_train_input_heating, y="Y",
                                                                        scalerX=RobustScalerX_heating,
                                                                        scalerY=RobustScalerY_heating,
                                                                        task="regression")

    # disposal data for cooling
    dtf_train_input_cooling = dtf_train_input.rename(columns={"coolingenergy": "Y"})
    dtf_train_input_cooling = dtf_train_input_cooling.drop('heatingenergy', axis=1)

    # data clusters of cooling
    dtf_train_input_cooling = add_feature_clusters(dtf_train_input_cooling, "hour",
                                                   dic_clusters_mapping=hour_clusters_cooling, dropx=False)
    dtf_train_input_cooling = add_dummies(dtf_train_input_cooling, x="hour_cluster", dropx=True)
    dtf_train_input_cooling = add_feature_clusters(dtf_train_input_cooling, "month",
                                                   dic_clusters_mapping=month_clusters_cooling, dropx=False)
    dtf_train_input_cooling = add_dummies(dtf_train_input_cooling, x="month_cluster", dropx=True)
    dtf_train_input_cooling = add_feature_clusters(dtf_train_input_cooling, "stories",
                                                   dic_clusters_mapping=city_building_stories_clusters_cooling,
                                                   dropx=False)
    dtf_train_input_cooling = add_dummies(dtf_train_input_cooling, x="stories_cluster", dropx=True)
    dtf_train_input_cooling = add_feature_clusters(dtf_train_input_cooling, "location",
                                                   dic_clusters_mapping=location_clusters, dropx=False)
    dtf_train_input_cooling = add_dummies(dtf_train_input_cooling, x="location_cluster", dropx=True)

    # drop data for cooling
    dtf_train_input_cooling = dtf_train_input_cooling.drop('hour', axis=1)
    dtf_train_input_cooling = dtf_train_input_cooling.drop('month', axis=1)
    dtf_train_input_cooling = dtf_train_input_cooling.drop('stories', axis=1)
    dtf_train_input_cooling = dtf_train_input_cooling.drop('location', axis=1)

    # Calculate the scaling parameters of cooling
    dtf_train_input_cooling, scalerX_cooling, scalerY_cooling = scaling(dtf_train_input_cooling, y="Y",
                                                                        scalerX=RobustScalerX_cooling,
                                                                        scalerY=RobustScalerY_cooling,
                                                                        task="regression")

    return dtf_train_input_heating, dtf_train_input_cooling, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling

# Use the GBR training model
def Train_surrogate_model(dtf_train_split, Model_training_variable):

    dtf_train_x = dtf_train_split[Model_training_variable].values
    dtf_train_y = dtf_train_split["Y"].values

    my_model = ensemble.GradientBoostingRegressor()

    param_dic = {'learning_rate':[0.15,0.1,0.05,0.01,0.005,0.001],      #weighting factor for the corrections by new trees when added to the model
                     'n_estimators':[100,250,500,750,1000,1250,1500,1750],  #number of trees added to the model
                     'max_depth':[2,3,4,5,6,7],                             #maximum depth of the tree
                     'min_samples_split':[2,4,6,8,10,20,40,60,100],         #sets the minimum number of samples to split
                     'min_samples_leaf':[1,3,5,7,9],                        #the minimum number of samples to form a leaf
                     'max_features':[2,3,4,5,6,7],                          #square root of features is usually a good starting point
                     'subsample':[0.7,0.75,0.8,0.85,0.9,0.95,1]}            #the fraction of samples to be used for fitting the individual base learners. Values lower than 1 gen

    my_model = tune_regr_model(dtf_train_x, dtf_train_y, my_model, param_dic, scoring = "r2", searchtype = "RandomSearch", n_iter = 100, cv = 5, figsize = (10,5))#应该是模型优选的一个过程，

    return my_model

# The trained model is tested using a segmented test set
def Train_surrogate_model_test(my_model, dtf_train_split, dtf_test_split, scalerY, Model_training_variable, HVAC):
    # Extract the input and output variables from the training and test data
    dtf_train_x = dtf_train_split[Model_training_variable].values
    dtf_train_y = dtf_train_split["Y"].values
    dtf_test_x = dtf_test_split[Model_training_variable].values
    dtf_test_y = dtf_test_split["Y"].values

    # Use the surrogate model to make predictions on the test data
    prediction = my_model.predict(dtf_test_x)
    # Transform the predicted and true output values to their original scale
    prediction = scalerY.inverse_transform(prediction.reshape(-1, 1)).reshape(-1)/1000
    true = dtf_test_y
    true = scalerY.inverse_transform(true.reshape(-1, 1)).reshape(-1)/1000
    # Calculate the R-squared score of the model's predictions
    score = metrics.r2_score(true, prediction)
    # Create a scatter plot to visualize the model's accuracy on the test data
    fig, ax = plt.subplots(figsize=(12,8))
    ax.scatter(prediction, true, lw=2, alpha=0.5, label='R2 = %0.2f' % (score))
    ax.plot([min(true), max(true)], [min(true), max(true)], linestyle='--', lw=2, color='black')
    ax.set_xlabel('Predicted (Unit : KJ/m\u00b2*h)')
    ax.set_ylabel('True (Unit : KJ/m\u00b2*h)')
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    plt.title('Accuracy detection on the test set')
    ax.legend(frameon=False)
    plt.subplots_adjust(left=0.1, bottom=0.1, right=0.95, top=0.93, wspace=0, hspace=0)
    # Save the scatter plot as a PNG file
    plt.savefig('picture/new/{hvac}_testset.png'.format(hvac = HVAC))
    #plt.show()

# For all the processes related to the training samples, only one parameter is left, that is, HVAC
def Train(HVAC):
    # Produce training input data
    # Train_input_data_produce(number)

    # Read and preprocess data for training
    dtf_train_input_heating, dtf_train_input_cooling, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling = Train_input_data_process()
    dtf_train_split_heating, dtf_test_split_heating = dtf_partitioning(dtf_train_input_heating, y="Y", test_size=0.3, shuffle=False)
    dtf_train_split_cooling, dtf_test_split_cooling = dtf_partitioning(dtf_train_input_cooling, y="Y", test_size=0.3, shuffle=False)

    if HVAC == 'heating':
        Model_training_variable = Model_training_variable_heating
        dtf_train_input, dtf_train_split, dtf_test_split, scalerY = dtf_train_input_heating, dtf_train_split_heating, dtf_test_split_heating, scalerY_heating
    elif HVAC == 'cooling':
        Model_training_variable = Model_training_variable_cooling
        dtf_train_input, dtf_train_split, dtf_test_split, scalerY = dtf_train_input_cooling, dtf_train_split_cooling, dtf_test_split_cooling, scalerY_cooling

    # Analyze the training input data
    # Train_input_data_sanalyze(dtf_train_input)

    # Train, test, and save surrogate models
    path_surrogate_model = './model_{hvac}.pkl'.format(hvac=HVAC)
    my_model = Train_surrogate_model(dtf_train_split, Model_training_variable)
    # my_model = joblib.load('./model_{hvac}_10000-51.pkl'.format(hvac=HVAC))
    Train_surrogate_model_test(my_model, dtf_train_split, dtf_test_split, scalerY, Model_training_variable, HVAC)
    joblib.dump(my_model, path_surrogate_model)

# Read and collate the predicted input data
def Predict_input_data_produce(w_scenario, e_path, year_observe, vintage, city_climate_station, city_climate_zone, city_building_stories):

    # Read the weather part and envelope part of the forecast input data
    path_weather_predict = './weather_scenario/{city_climate_station}_weather_site/{city_climate_station}-{w_scenario}-{year}.csv'.format(
        w_scenario = w_scenario, year = year_observe, city_climate_station = city_climate_station)
    path_envelope_predict = './envelope_path/{city_climate_zone}/{city_climate_zone}-{city_building_stories}-{e_path}-{year}-{age}-envelope.csv'.format(
        e_path = e_path, year = year_observe, age = int(vintage), city_climate_zone = city_climate_zone, city_building_stories = city_building_stories)
    dtf_predict_input_envelope = pd.read_csv(path_envelope_predict)
    dtf_predict_input_envelope = dtf_predict_input_envelope.drop('Unnamed: 0', axis=1)
    dtf_predict_input_weather = pd.read_csv(path_weather_predict)

    # The merging of two parts of data
    dtf_predict_input = pd.concat([dtf_predict_input_envelope, dtf_predict_input_weather], axis=1, join='inner')
    dtf_predict_input = dtf_predict_input.drop('Unnamed: 0', axis=1)
    # Returns the processed merged data
    return dtf_predict_input

# Standardization processes predictive input data
def Predict_input_data_process(city_building_hvac, scalerX, dtf_predict_input):
    # Due to the singularity of climate stations(has been pre-written) and stories, the standardized processing of prediction input data is different from that of training input data
    # hours
    hour_clusters_heating = {"max": [], "mean": [20, 21, 22, 23, 24, 11, 12], "min": [13, 14, 15, 16, 17, 18, 19]}
    hour_clusters_cooling = {"max": [14, 15, 16, 17, 18, 19, 20, 21], "mean": [11, 12, 13, 22, 23, 24], "min": []}
    # months
    month_clusters_heating = {"max": [1, 12], "mean": [2, 11], "min": []}
    month_clusters_cooling = {"max": [6, 7, 8], "mean": [5, 9], "min": []}

    # Check if HVAC system is heating or cooling
    if city_building_hvac == 'heating':
        # If heating, use heating-specific model training variables
        Model_training_variable = Model_training_variable_heating

        # Cluster hours of the day and add dummies for each cluster
        dtf_predict_input_heating = add_feature_clusters(dtf_predict_input, "hour",
                                                         dic_clusters_mapping=hour_clusters_heating, dropx=False)
        dtf_predict_input_heating = add_dummies(dtf_predict_input_heating, x="hour_cluster", dropx=True)

        # Cluster months of the year and add dummies for each cluster
        dtf_predict_input_heating = add_feature_clusters(dtf_predict_input_heating, "month",
                                                         dic_clusters_mapping=month_clusters_heating, dropx=False)
        dtf_predict_input = add_dummies(dtf_predict_input_heating, x="month_cluster", dropx=True)

        # Create a new DataFrame for the building stories and fill with appropriate cluster values based on number of stories
        if dtf_predict_input['stories'][0] == 1:
            data_city_building_stories = pd.concat(
                [pd.DataFrame(np.repeat(0, 8760, axis=0)), pd.DataFrame(np.repeat(0, 8760, axis=0))], axis=1)
            data_city_building_stories.columns = ['stories_cluster_mean', 'stories_cluster_min']
        elif dtf_predict_input['stories'][0] == 2 or dtf_predict_input['stories'][0] == 3:
            data_city_building_stories = pd.concat(
                [pd.DataFrame(np.repeat(1, 8760, axis=0)), pd.DataFrame(np.repeat(0, 8760, axis=0))], axis=1)
            data_city_building_stories.columns = ['stories_cluster_mean', 'stories_cluster_min']
        else:
            data_city_building_stories = pd.concat(
                [pd.DataFrame(np.repeat(0, 8760, axis=0)), pd.DataFrame(np.repeat(1, 8760, axis=0))], axis=1)
            data_city_building_stories.columns = ['stories_cluster_mean', 'stories_cluster_min']

    elif city_building_hvac == 'cooling':
        # If cooling, use cooling-specific model training variables
        Model_training_variable = Model_training_variable_cooling

        # Cluster hours of the day and add dummies for each cluster
        dtf_predict_input_cooling = add_feature_clusters(dtf_predict_input, "hour",
                                                         dic_clusters_mapping=hour_clusters_cooling, dropx=False)
        dtf_predict_input_cooling = add_dummies(dtf_predict_input_cooling, x="hour_cluster", dropx=True)

        # Cluster months of the year and add dummies for each cluster
        dtf_predict_input_cooling = add_feature_clusters(dtf_predict_input_cooling, "month",
                                                         dic_clusters_mapping=month_clusters_cooling, dropx=False)
        dtf_predict_input = add_dummies(dtf_predict_input_cooling, x="month_cluster", dropx=True)

        # Create a new DataFrame for the building stories and fill with appropriate cluster values based on number of stories
        if dtf_predict_input['stories'][0] == 1:
            data_city_building_stories = pd.DataFrame(np.repeat(1, 8760, axis=0))
            data_city_building_stories.columns = ['stories_cluster_min']
        else:
            data_city_building_stories = pd.DataFrame(np.repeat(0, 8760, axis = 0))
            data_city_building_stories.columns = ['stories_cluster_min']

    dtf_predict_input = pd.concat([dtf_predict_input, data_city_building_stories], axis = 1)
    dtf_predict_input = dtf_predict_input.drop('hour', axis = 1)
    dtf_predict_input = dtf_predict_input.drop('month', axis = 1)
    dtf_predict_input = dtf_predict_input.drop('stories', axis = 1)
    dtf_predict_input = dtf_predict_input.drop('location', axis = 1)

    # Standardized treatment
    dtf_predict_input_standard = scalerX.transform(dtf_predict_input)
    dtf_predict_input_standard = pd.DataFrame(dtf_predict_input_standard, columns = dtf_predict_input.columns, index = dtf_predict_input.index)
    dtf_predict_input = dtf_predict_input_standard[Model_training_variable].values

    return dtf_predict_input

# Categorize into three kinds for prediction
class Predict_surrogate_model:
    # Core model
    surrogate_model_heating = joblib.load('./model_heating.pkl')
    surrogate_model_cooling = joblib.load('./model_cooling.pkl')

    # The first type of prediction has the same city results directly replicated
    def Copy_predicted(self, city_code_predicted, city_code, city_building_hvacs):
        for city_building_hvac in city_building_hvacs:
            for e_path in Envelope_Paths:
                for w_scenario in Weather_Scenarios:
                    path_city_predicted = './energy/{city_code}/{city_code}-{Envelope_Path}-{Weather_Scenario}-{city_building_hvac}-tranist.xls'.format(
                        city_code = city_code_predicted, Envelope_Path = e_path, Weather_Scenario = w_scenario, city_building_hvac = city_building_hvac)
                    path_city = './energy/{city_code}/{city_code}-{Envelope_Path}-{Weather_Scenario}-{city_building_hvac}-tranist.xls'.format(
                        city_code = city_code, Envelope_Path = e_path, Weather_Scenario = w_scenario, city_building_hvac = city_building_hvac)
                    File_SaveAs(path_city_predicted, path_city)

    # The second type of forecasting is city forecasting with only one way of using energy
    def Single_predict(self,city_code, e_path, w_scenario, city_climate_station, city_building_stories, city_climate_zone, city_building_ages, city_building_hvac, city_building_hvac_time, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling):
        dtf_predict_output_years_ages = []

        for year_observe in range(2020, 2101):
            print(e_path, w_scenario, year_observe)
            for vintage in city_building_ages:
                # Standardized processing of predictive input data
                dtf_predict_input = Predict_input_data_produce(w_scenario, e_path, year_observe, vintage, city_climate_station, city_climate_zone, city_building_stories)

                # The surrogate model is invoked energetically to predict
                if city_building_hvac == 'heating':
                    dtf_predict_input = Predict_input_data_process(city_building_hvac, scalerX_heating, dtf_predict_input)
                    dtf_predict_output = self.surrogate_model_heating.predict(dtf_predict_input)
                    dtf_predict_output = scalerY_heating.inverse_transform(dtf_predict_output.reshape(-1, 1)).reshape(-1)
                    dtf_predict_output_year_age_hours_heating = pd.DataFrame(dtf_predict_output)
                elif city_building_hvac == 'cooling':
                    dtf_predict_input = Predict_input_data_process(city_building_hvac, scalerX_cooling, dtf_predict_input)
                    dtf_predict_output = self.surrogate_model_cooling.predict(dtf_predict_input)
                    dtf_predict_output = scalerY_cooling.inverse_transform(dtf_predict_output.reshape(-1, 1)).reshape(-1)
                    dtf_predict_output_year_age_hours_cooling = pd.DataFrame(dtf_predict_output)

                # The annual energy intensity is calculated by way of using energy
                dtf_predict_output_year_ages = []
                if city_building_hvac == 'heating':
                    dtf_predict_output_year_age = pd.concat([dtf_predict_output_year_age_hours_heating[:int(city_building_hvac_time[0])+1], dtf_predict_output_year_age_hours_heating[int(city_building_hvac_time[1]):]], axis = 0, join='inner')
                elif city_building_hvac == 'cooling':
                    dtf_predict_output_year_age = dtf_predict_output_year_age_hours_cooling[int(city_building_hvac_time[0]): int(city_building_hvac_time[1])+1]
                dtf_predict_output_year_age[dtf_predict_output_year_age < 1000] = 0

                # Processing and storage of predictive output data
                dtf_predict_output_year_ages.append(dtf_predict_output_year_age.apply(lambda x: x.sum(), axis = 0).iloc[0])
                dtf_predict_output_year_ages.append(year_observe)
                dtf_predict_output_year_ages.append(vintage)
                dtf_predict_output_years_ages.append(dtf_predict_output_year_ages)
        dtf_predict_output_years_ages = pd.DataFrame(dtf_predict_output_years_ages)
        dtf_predict_output_years_ages.to_excel('./energy/{city_code}/{city_code}-{e_path}-{w_scenario}-{city_building_hvac}-tranist.xls'.format(city_code = city_code, e_path = e_path, w_scenario = w_scenario,city_building_hvac = city_building_hvac))

    # The third type of forecast is the city forecast which is used in both ways
    def Multiple_predict(self,city_code, e_path, w_scenario, city_climate_station, city_building_stories, city_climate_zone, city_building_ages, city_building_hvac_time, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling):
        dtf_predict_output_years_ages_heating = []
        dtf_predict_output_years_ages_cooling = []

        for year_observe in range(2020, 2101):
            print(e_path, w_scenario, year_observe)
            for vintage in city_building_ages:
                # Standardized processing of predictive input data
                dtf_predict_input = Predict_input_data_produce(w_scenario, e_path, year_observe, vintage, city_climate_station, city_climate_zone, city_building_stories)
                dtf_predict_output_year_ages_heating = []
                dtf_predict_output_year_ages_cooling = []

                # The surrogate model is invoked energetically to predict
                dtf_predict_input_heating = Predict_input_data_process('heating', scalerX_heating, dtf_predict_input)
                dtf_predict_output_heating = self.surrogate_model_heating.predict(dtf_predict_input_heating)
                dtf_predict_output_heating = scalerY_heating.inverse_transform(dtf_predict_output_heating.reshape(-1, 1)).reshape(-1)
                dtf_predict_output_year_age_hours_heating = pd.DataFrame(dtf_predict_output_heating)
                dtf_predict_input_cooling = Predict_input_data_process('cooling', scalerX_cooling, dtf_predict_input)
                dtf_predict_output_cooling = self.surrogate_model_cooling.predict(dtf_predict_input_cooling)
                dtf_predict_output_cooling = scalerY_cooling.inverse_transform(dtf_predict_output_cooling.reshape(-1, 1)).reshape(-1)
                dtf_predict_output_year_age_hours_cooling = pd.DataFrame(dtf_predict_output_cooling)

                # The annual energy intensity is calculated by way of using energy
                dtf_predict_output_year_age_heating = pd.concat([dtf_predict_output_year_age_hours_heating[:int(city_building_hvac_time[0])+1], dtf_predict_output_year_age_hours_heating[int(city_building_hvac_time[1]):]], axis = 0, join = 'inner')
                dtf_predict_output_year_age_heating[dtf_predict_output_year_age_heating < 1000] = 0
                dtf_predict_output_year_age_cooling = dtf_predict_output_year_age_hours_cooling[int(city_building_hvac_time[2]): int(city_building_hvac_time[3]) + 1]
                dtf_predict_output_year_age_cooling[dtf_predict_output_year_age_cooling < 1000] = 0

                # Processing and storage of predictive output data
                dtf_predict_output_year_ages_heating.append(dtf_predict_output_year_age_heating.apply(lambda x: x.sum(),axis = 0).iloc[0])
                dtf_predict_output_year_ages_heating.append(year_observe)
                dtf_predict_output_year_ages_heating.append(vintage)
                dtf_predict_output_years_ages_heating.append(dtf_predict_output_year_ages_heating)
                dtf_predict_output_year_ages_cooling.append(dtf_predict_output_year_age_cooling.apply(lambda x: x.sum(), axis = 0).iloc[0])
                dtf_predict_output_year_ages_cooling.append(year_observe)
                dtf_predict_output_year_ages_cooling.append(vintage)
                dtf_predict_output_years_ages_cooling.append(dtf_predict_output_year_ages_cooling)
        dtf_predict_output_years_ages_heating = pd.DataFrame(dtf_predict_output_years_ages_heating)
        dtf_predict_output_years_ages_heating.to_excel('./energy/{city_code}/{city_code}-{e_path}-{w_scenario}-heating-tranist.xls'.format(city_code = city_code, e_path = e_path, w_scenario = w_scenario))
        dtf_predict_output_years_ages_cooling = pd.DataFrame(dtf_predict_output_years_ages_cooling)
        dtf_predict_output_years_ages_cooling.to_excel('./energy/{city_code}/{city_code}-{e_path}-{w_scenario}-cooling-tranist.xls'.format(city_code = city_code, e_path = e_path, w_scenario = w_scenario))

# The building energy consumption and building energy intensity of each year after prediction were calculated
def Predict_output_data_calculate(city_code, e_path, w_scenario, city_building_hvac, city_building_ages):
    # This has to be written row by column in excel
    workbook = xlwt.Workbook(encoding='utf-8')
    worksheet = workbook.add_sheet('My Worksheet')
    # Read the transfer data, is a summary of the data.
    path_tranist = './energy/{city_code}/{city_code}-{e_path}-{w_scenario}-{city_building_hvac}-tranist.xls'.format(city_code = city_code, e_path = e_path, w_scenario = w_scenario, city_building_hvac = city_building_hvac)
    dtf_predict_tranist = pd.read_excel(path_tranist)

    # The predicted annual building energy intensity is arranged in the previous building stock format
    row = 0
    for i in range(len(city_building_ages)):
         # Building energy consumption for all observed years during the construction period
        energy_intensity_years_age = dtf_predict_tranist[dtf_predict_tranist[2] == city_building_ages[i]][0].to_list()
        # The building energy intensity of each year is written according to the situation of different built ages
        if i < len(city_building_ages) - 1:
            for j in range(city_building_ages[i + 1]-city_building_ages[i]):
                for col in range(81):
                    worksheet.write(row, col, label = energy_intensity_years_age[col])
                row = row + 1
        elif i == len(city_building_ages) - 1 and city_building_ages[-1] <= 2020:
            for j in range(2021-city_building_ages[-1]):
                for col in range(81):
                    worksheet.write(row, col, label = energy_intensity_years_age[col])
                row = row + 1
            # The starting point for the flow of new construction after 2020
            new_building = 1
            for j in range(2101 - 2021):
                for col in range(new_building, 81):
                    worksheet.write(row, col, label = energy_intensity_years_age[col])
                new_building = new_building + 1
                row = row + 1

        # You have to save it before you can read all the data
        workbook.save('./energy/{city_code}/{city_code}-{Envelope_Path}-{Weather_Scenario}-{city_building_hvac}-stock.xls'.format(city_code = city_code, Envelope_Path = e_path, Weather_Scenario = w_scenario, city_building_hvac = city_building_hvac))
    dtf_intensity_years_ages = pd.read_excel('./energy/{city_code}/{city_code}-{Envelope_Path}-{Weather_Scenario}-{city_building_hvac}-stock.xls'.format(city_code = city_code, Envelope_Path = e_path, Weather_Scenario = w_scenario, city_building_hvac = city_building_hvac))
    dtf_building_years_ages = pd.read_csv('./building_stock/n{city_code}_dynamic_stock.csv'.format(city_code = city_code))
    dtf_intensity_years_ages.columns = year_observes
    dtf_building_years_ages = dtf_building_years_ages.drop('age', axis = 1)
    dtf_building_years_ages.columns = year_observes

    # First, the energy consumption of all built ages in each year is calculated, and then the total energy consumption of each year is summed up, and then the total area of each year is divided to obtain the energy intensity of each year.
    dtf_energy_years_ages = dtf_intensity_years_ages * dtf_building_years_ages
    energy_year = dtf_energy_years_ages.apply(lambda x: x.sum(), axis = 0)
    area_year = dtf_building_years_ages.apply(lambda x: x.sum(), axis = 0)
    intensity_year = energy_year / area_year

    return energy_year, intensity_year

# All the functions related to the prediction are integrated, leaving only one parameter, the city number
def Predict(city_number):

    my_predict = Predict_surrogate_model()

    # Consistent data and data preprocessing are used for prediction and training
    dtf_train_input_heating, dtf_train_input_cooling, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling = Train_input_data_process()

    # Read the information for each city from the master table
    city_code = city_information_table.iloc[city_number - 1, 0]
    city_climate_station = city_information_table.iloc[city_number - 1, 1]
    city_climate_zone = city_information_table.iloc[city_number - 1, 4]
    city_building_stories = city_information_table.iloc[city_number - 1, 7]
    city_building_ages_total = city_information_table.iloc[city_number - 1, 8 : 22].tolist()
    city_building_hvac_index = city_information_table.iloc[city_number - 1, 22]
    city_building_hvac_time_heating = city_information_table.iloc[city_number - 1, 23 : 25].tolist()
    city_building_hvac_time_cooling = city_information_table.iloc[city_number - 1, 25 : 27].tolist()

    print(city_code)
    if city_climate_zone == 'warm':
        return

    # According to the information of each city to synthesize the required forecast input data, output and collate the forecast results
    energy_years = []
    intensity_years = []
    ouput_scenario_indexs = []

    # Determine whether it has been predicted by the model, or simply by replicating previously predicted results
    is_predict = 0

    # Counties with the same climate and building information directly copy the predicted results
    for city_number_from_0 in range(city_number):

        # Check it backwards, because the same information is usually near to each other
        city_number_predicted = city_number - 1 - city_number_from_0

        # Judging from three dimensions: city_climate_station、 city_climate_zone、 city_building_stories
        if city_number_predicted < city_number - 1  and city_climate_station == city_information_table.iloc[city_number_predicted  , 1] \
                                and city_climate_zone == city_information_table.iloc[city_number_predicted , 4]\
                                and city_building_stories == city_information_table.iloc[city_number_predicted  , 7]:
            print('copy!')
            city_code_predicted = city_information_table.iloc[city_number_predicted , 0]
            if int(city_building_hvac_index) == 2:
                city_building_hvacs = ['heating', 'cooling']
            else:
                city_building_hvacs = [['heating', 'cooling'][int(city_building_hvac_index)]]
            my_predict.Copy_predicted(city_code_predicted, city_code, city_building_hvacs)

            # after copy, is_predict = 1 , no need to predict, and break
            is_predict = 1
            break

    # Counties that do not have the same information use the surrogate model for the first forecast
    if is_predict == 0:
        for i in range(len(Envelope_Paths)):

            # Different envelope paths correspond to several building ages
            city_building_ages = city_building_ages_total[[0, 3, 6][i] : [3, 6, 14][i]]
            city_building_ages = [int(i) for i in city_building_ages]
            while 0 in city_building_ages:
                city_building_ages.remove(0)

            for j in range(len(Weather_Scenarios)):
                print(Envelope_Paths[i] + '-' + Weather_Scenarios[j])
                if int(city_building_hvac_index) == 2:
                    city_building_hvacs = ['heating', 'cooling']
                    city_building_hvac_time = city_building_hvac_time_heating + city_building_hvac_time_cooling
                    my_predict.Multiple_predict(city_code, Envelope_Paths[i], Weather_Scenarios[j], city_climate_station, city_building_stories, city_climate_zone, city_building_ages, city_building_hvac_time, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling)
                else:
                    city_building_hvacs = [['heating', 'cooling'][int(city_building_hvac_index)]]
                    city_building_hvac_time = [city_building_hvac_time_heating, city_building_hvac_time_cooling][int(city_building_hvac_index)]
                    my_predict.Single_predict(city_code, Envelope_Paths[i], Weather_Scenarios[j], city_climate_station, city_building_stories, city_climate_zone, city_building_ages, city_building_hvacs.iloc[0], city_building_hvac_time, scalerX_heating, scalerY_heating, scalerX_cooling, scalerY_cooling)

    # The prediction results are calculated by scenarios
    for i in range(len(Envelope_Paths)):
        city_building_ages = city_building_ages_total[[0, 3, 6][i]: [3, 6, 14][i]]
        city_building_ages = [int(i) for i in city_building_ages]
        while 0 in city_building_ages:
            city_building_ages.remove(0)
        for j in range(len(Weather_Scenarios)):
            for city_building_hvac in city_building_hvacs:
                energy_year, intensity_year = Predict_output_data_calculate(city_code, Envelope_Paths[i], Weather_Scenarios[j], city_building_hvac, city_building_ages)
                ouput_scenario_indexs.append(city_building_hvac + '-' + Envelope_Paths[i] + '-' + Weather_Scenarios[j])
                energy_years.append(energy_year)
                intensity_years.append(intensity_year)

    # The output is the result of time series under nine scenarios
    intensity_years = pd.DataFrame(intensity_years)
    intensity_years.index = ouput_scenario_indexs
    energy_years = pd.DataFrame(energy_years)
    energy_years.index = ouput_scenario_indexs
    energy_years.to_csv('./energy/{city_code}/{city_code}-energys.csv'.format(city_code = city_code))
    intensity_years.to_csv('./energy/{city_code}/{city_code}-intensitys.csv'.format(city_code = city_code))

# Record start time
start = time.time()

# Sample(10000)
# Call scope, which must start at 1
'''
for sample_order in range(1,2): # Range of all samples, a total of 5000 samples must start at 1
    print(sample_order)
    Simulate(sample_order)
'''

# Train('heating')

'''
for n in range(1, 1678):
    Predict(n) # Add a time sleep
    time.sleep(0)
'''

# Record end time
end = time.time()

# Print elapsed time
print('Time taken:', end - start)


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

histogram_path1 = "histogram_kitti.npy"
histogram_path2 = "histogram_middlebury.npy"

MIN_DEPTH = 1e-3
MAX_DEPTH = 80

n = 200
display_bins = 200#int(55/80*n)

histogram1 = np.load(histogram_path1)
histogram2 = np.load(histogram_path2)

fig = plt.figure()
ax = fig.add_subplot(111)
xs = np.histogram_bin_edges([], bins=n, range=(MIN_DEPTH, MAX_DEPTH))[:-1][:display_bins]
width = (MAX_DEPTH-MIN_DEPTH)/n

ax.bar(xs, histogram1[:display_bins], width=width, alpha=0.5)
ax.bar(xs, histogram2[:display_bins], width=width, alpha=0.5)
ax.legend(["Kitti", "Middlebury"])
ax.set_xlabel("Depth")

ax.yaxis.set_major_formatter(ticker.PercentFormatter(xmax=1))

plt.savefig("histogram_combined.png")


"""
Mark Porath, Dec 25

the result is ok, but not great -- see description in cloud console;
the python library 're' is useful, but beam.io.ReadFromText is too limited;
beam.dataframe module is supposed to make csv sources easy, but I found a bug:
to_pcollection fails when dataframes have nulls -- fix is weeks away, beam 2.36

practice-springml:nyc19_dataset_25.bnb_table
practice-springml:nyc19_dataset_25.listings_per_hood
"""

import re, logging
import apache_beam as beam
# from apache_beam.dataframe.io import read_csv
# from apache_beam.dataframe.convert import to_pcollection
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.io import ReadFromText, WriteToBigQuery

options = PipelineOptions([
    '--runner', 'DataflowRunner',
    '--project', 'practice-springml',
    # '--template_location', 'gs://practice-job/template',
    '--temp_location', 'gs://practice-springml.appspot.com/tmp',
    '--region', 'us-central1',
    '--save_main_session'
    ])

def format_bnb(row):

    fields = [
        'id',
        'name',
        'host_id',
        'host_name',
        'neighbourhood_group',
        'neighbourhood',
        'latitude',
        'longitude',
        'room_type',
        'price',
        'minimum_nights',
        'number_of_reviews',
        'last_review',
        'reviews_per_month',
        'calculated_host_listings_count',
        'availability_365'
        ]

    p = re.compile(r"""(?xs:(\d+),(.*),(\d{4,9}),(.*),
                        ([BMQS].*),  # the 5 boroughs
                        ([A-Z]\D*),  # neighbourhood starts with capital
                        (40\.\d+),   # latitude (40.49979, 40.91306)
                        (-\d+\.\d+), # longitude (-74.24442, -73.71299)
                        (.*),(\d+),(\d+),(\d+),
                        (.*),(.*),   # last_review, reviews_per_month
                        (\d+),(\d+))""")

    m = p.findall(row)

    # COULD PRACTICE DEADLETTER PATTERN HERE (not implemented yet);
    # ERRORS HAVE NEWLINE CHARACTER IN 'NAME'
    if len(m) == 0:
        nulls = [None] * 16
        return { k:v for k,v in zip(fields, nulls) } #yield?

    # cast to int
    m = list(m[0])
    for i in [0,2,9,10,11,14,15]:
        m[i] = int(m[i])

    # cast to float; column 13, 'reviews_per_month', contains Nulls
    for i in [6,7,13]:
        if len(m[i]) > 0:
            m[i] = float(m[i])
        else:
            m[i] = None

    # convert to BQ date; column 12, 'last_review', contains Nuls
    if len(m[12]) > 0:
        month,day,year = m[12].split('/')
        m[12] = year+'-'+month.zfill(2)+'-'+day.zfill(2)
    else:
        m[12] = None

    return { k:v for k,v in zip(fields, m) }

def tally_nhood(listing):
    if listing:
        return (listing['neighbourhood'], 1)

def format_tally(nhood_count):
    return {'neighbourhood':nhood_count[0], 'n_listings':nhood_count[1]}

def run(options = options):

    schema0 = [
        "id:INTEGER",
        "name:STRING",
        "host_id:INTEGER",
        "host_name:STRING",
        "neighbourhood_group:STRING",
        "neighbourhood:STRING",
        "latitude:FLOAT",
        "longitude:FLOAT",
        "room_type:STRING",
        "price:INTEGER",
        "minimum_nights:INTEGER",
        "number_of_reviews:INTEGER",
        "last_review:DATE",
        "reviews_per_month:FLOAT",
        "calculated_host_listings_count:INTEGER",
        "availability_365:INTEGER"]
    schema1 = ",".join([f for f in schema0])

    gcs_csv = "gs://airbnb-nyc-19/AB_NYC_2019.csv"

    p = beam.Pipeline(options = options)

    pcoll = (
        p
        | 'read_csv' >> ReadFromText(gcs_csv, skip_header_lines=1)
        | 'format_bnb' >> beam.Map(format_bnb)
        )

    (pcoll
    | 'write1' >> WriteToBigQuery(
        table = "nyc19_dataset_25.bnb_table",
        schema = schema1,
        create_disposition = 'CREATE_IF_NEEDED',
        write_disposition = 'WRITE_TRUNCATE'))

    (pcoll
    | 'nhood_is_one' >> beam.Map(tally_nhood)
    | 'nhood_counts' >> beam.CombinePerKey(sum)
    | 'format_tally' >> beam.Map(format_tally)
    | 'write2' >> WriteToBigQuery(
        table = "nyc19_dataset_25.listings_per_nhood",
        schema = 'neighbourhood:STRING,n_listings:INTEGER',
        create_disposition = 'CREATE_IF_NEEDED',
        write_disposition = 'WRITE_TRUNCATE'))

    p.run().wait_until_finish()

if __name__ == '__main__':
    logging.getLogger().setLevel(logging.INFO)
    run()


import threading
import time
import numpy
import cv2
import math

class LineFinder (threading.Thread):
    def __init__(self, img):
        threading.Thread.__init__(self)
        self.img = img

    def run(img):
        print("running line detection thread")
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#        gray = cv2.equalizeHist(gray)
#        gray = cv2.GaussianBlur(gray, (9,9), 0)

        edged = cv2.Canny(gray, 50, 150, apertureSize = 3)
        lines = cv2.HoughLines(edged, 1, numpy.pi/180.0, 100, numpy.array([]), 0, 0)

#        for x1, y1, x2, y2 in lineThread[0]:
#            cv2.line(img, (x1,y1),(x2,y2),(0,255,0),2)

        a,b,c = lines.shape
        for i in range(a):
            rho = lines[i][0][0]
            theta = lines[i][0][1]
            a = math.cos(theta)
            b = math.sin(theta)
            x0, y0 = a*rho, b*rho
            pt1 = (int(x0+1000*(-b)), int(y0+1000*(a)))
            pt2 = (int(x0-1000*(-b)), int(y0-1000*(a)))
            cv2.line(img, pt1, pt2, (0,0,255), 2, cv2.LINE_AA)

        print("exiting line detection thread")


import pandas as pd

import numpy as np

netflix_data = pd.read_csv("Netflix_titles.csv", index_col = 0)
pd.options.display.max_columns = None
pd.options.display.max_rows = None
pd.set_option('display.max_colwidth', None)

#drop the director.

netflix_data=netflix_data.drop(["director"],axis=1)
# finding total number of null values
print(netflix_data .isnull().sum())


import os

from .common import Common, BASE_DIR

class Test(Common):
    DEBUG = True

    # Testing
    INSTALLED_APPS = Common.INSTALLED_APPS
    INSTALLED_APPS += ("django_nose",)
    TEST_RUNNER = "boxme_product.libs.runners.TestSuiteRunner"
    NOSE_ARGS = [
        BASE_DIR,
        "--nocapture",
        "--nologcapture",
        "--logging-clear-handlers",
    ]

    MIGRATION_MODULES = {
        "admin": None,
        "auth": None,
        "contenttypes": None,
        "sessions": None,
        "boxme_core": None,
        "boxme_countries": None,
        "boxme_jwt": None,
        "ns_addresses": None,
        "ns_fees": None,
        "ns_orders": None,
        "ns_products": None,
        "ns_purchase_orders": None,
        "ns_stores": None,
    }

    MONGODB = "178.128.23.178"
    MONGODB_NAME = f"{Common.MONGODB_NAME}_test"

    LOCAL_STATIC_DIRS = os.path.join(BASE_DIR, "storage")


import socket
import time
import threading

clients = [] # clients 리스트

def chat(sock: socket):
    while True:
        data = sock.recv(1024) # 메시지 수신

        if 'quit' in data.decode(): # quit 입력 시
            print(sock, 'exited')
            clients.remove(sock) # 해당 client 삭제
            continue

        print(time.asctime() + str(sock.getsockname()) + ':' + data.decode()) # client 메시지 log 출력

        for client in clients: # clients 리스트 순회
            if client != sock: # 송신 client를 제외한 나머지 client에게
                client.send(data) # 메시지 송신

    sock.close()

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # IPv4, TCP
s.bind(('', 2500))
s.listen(5)

print('Server Started')

while True:
    conn, addr = s.accept()
    if addr not in clients: # 신규 client일 때
        clients.append(conn) # client 목록에 추가
        print('connected by', addr)
        th = threading.Thread(target=chat, args=(conn, )) # 멀티스레드를 활용하여 송수신
        th.start()


# Open the ngramoutput.txt file in read mode
with open("ngramoutput.txt", "r") as ngram_file:
    # Read each line of the file
    for line in ngram_file:
        # Open the wordlist.txt file in read mode
        with open("wordlist.txt", "r") as wordlist_file:
            # Read each word in the wordlist
            for word in wordlist_file:
                # Check if the line from ngramoutput.txt exists in the word from wordlist.txt
                if line in word:
                    # If it does, get the position of the line in the word
                    position = word.index(line)
                    # Print the position and the line from ngramoutput.txt to the output file
                    with open("output.txt", "a") as output_file:
                        output_file.write(f"[{position}] {line}")