""" Write Data Inventory to Disk as TFRecord files """
import random
import os
import math
import time
import logging
import textwrap
from multiprocessing import Process, Manager

import tensorflow as tf

from camera_trap_classifier.data.image import (
    read_image_from_disk_and_convert_to_jpeg)
from camera_trap_classifier.data.utils import (
    slice_generator, estimate_remaining_time)

tf.enable_eager_execution()

logger = logging.getLogger(__name__)

class DatasetWriter(object):
    def __init__(self, tfr_encoder):
        self.tfr_encoder = tfr_encoder
        self.files = dict()

    def encode_to_tfr(
         self, tfrecord_dict,
         output_dir,
         file_prefix,
         image_root_path=None,
         image_pre_processing_fun=None,
         image_pre_processing_args=None,
         random_shuffle_before_save=True,
         overwrite_existing_files=True,
         max_records_per_file=None,
         write_tfr_in_parallel=False,
         process_images_in_parallel=False,
         process_images_in_parallel_size=100,
         processes_images_in_parallel_n_processes=4):
        """ Export TFRecord Dict to a TFRecord file """

        self.tfrecord_dict = tfrecord_dict
        self.image_pre_processing_fun = image_pre_processing_fun
        self.image_pre_processing_args = image_pre_processing_args
        self.random_shuffle_before_save = random_shuffle_before_save
        self.file_prefix = file_prefix
        self.image_root_path = image_root_path
        self.files[file_prefix] = list()
        self.write_tfr_in_parallel = write_tfr_in_parallel
        self.process_images_in_parallel = process_images_in_parallel
        self.process_images_in_parallel_size = process_images_in_parallel_size
        self.processes_images_in_parallel_n_processes = \
            processes_images_in_parallel_n_processes

        logger.info("Starting to Encode Data to TFRecords")

        if not isinstance(tfrecord_dict, dict):
            logger.error("tfrecord_dict must be a dictionary")
            raise ValueError("tfrecord_dict must be a dictionary")

        # Sort records to ensure the records are split into files
        # equally each time
        record_ids = list(tfrecord_dict.keys())
        record_ids.sort()
        n_records = len(tfrecord_dict.keys())

        logger.info("Start Writing Records to TFRecord-File - Total %s" %
                    n_records)

        # Generate output file names
        if max_records_per_file is None:
            n_files = 1
        else:
            n_files = math.ceil(n_records / max_records_per_file)

        output_paths = list()
        for i in range(0, n_files):
            file_name = '%s_%03d-of-%03d.tfrecord' % (file_prefix, i+1, n_files)
            output_paths.append(os.path.join(*[output_dir, file_name]))

        # processes list if parallel processing is enabled
        if self.write_tfr_in_parallel:
            processes_list = list()

        slices = slice_generator(n_records, n_files)

        # Write each file
        for f_id, (start_i, end_i) in enumerate(slices):
            output_file = output_paths[f_id]
            # generate record slices for each file
            file_record_ids = record_ids[start_i:end_i]

            # check if file already exists
            file_exists = os.path.exists(output_file)

            if file_exists and not overwrite_existing_files:
                logger.info("File: %s exists - not gonna overwrite" %
                            output_file)
                self.files[file_prefix].append(output_file)
            else:
                if self.write_tfr_in_parallel:
                    pr = Process(target=self._write_to_file,
                                 args=(output_file, file_record_ids))
                    pr.start()
                    processes_list.append(pr)
                else:
                    if self.process_images_in_parallel:
                        self._write_to_file_parallel(output_file,
                                                     file_record_ids)
                    else:
                        self._write_to_file(output_file, file_record_ids)
            self.files[self.file_prefix].append(output_file)
        # start all processes
        if self.write_tfr_in_parallel:
            for p in processes_list:
                p.join()

    def _serialize_record_batch(self, record_batch, outlist):
        """ Serialize a list of records """
        for record_data in record_batch:
            serialized_record = self._serialize_record(record_data)
            if serialized_record is not None:
                outlist.append(serialized_record)

    def _read_image_from_disk(self, image_path_full):
        """ Read Image from Disk """
        if self.image_pre_processing_fun is not None:
            self.image_pre_processing_args['path_to_image'] = \
                image_path_full
            image_raw = self.image_pre_processing_fun(
                 **self.image_pre_processing_args)
        else:
            image_raw = \
                read_image_from_disk_and_convert_to_jpeg(image_path_full)
        return image_raw

    def _serialize_record(self, record_data):
        """ Serialize a single record """
        # Process all images in a record
        raw_images = list()
        for image_path in record_data['image_paths']:
            # Create image path
            if self.image_root_path is not None:
                image_path_full = os.path.join(self.image_root_path,
                                               image_path.lstrip(os.sep))
            else:
                image_path_full = image_path
            try:
                image_raw = self._read_image_from_disk(image_path_full)
            except Exception as e:
                logger.debug("Failed to read image: %s , error %s" %
                             (image_path_full, str(e)))
                continue

            raw_images.append(image_raw)

        # check if at least one image is available
        if len(raw_images) == 0:
            return None

        record_data['images'] = raw_images

        serialized_record = self.tfr_encoder(record_data)

        return serialized_record

    def _write_to_file(self, output_file, record_ids):
        """ Write a TFR File """

        # Create and Write Records to TFRecord file
        logger.info("Start Writing %s" % output_file)
        n_records = len(record_ids)
        successfull_writes = 0
        start_time = time.time()

        # Randomly shuffle records before saving, this is better for
        # model training
        if self.random_shuffle_before_save:
            random.seed(123)
            random.shuffle(record_ids)

        # temporary filename for writing to avoid complications after
        # a write is incomplete

        output_temp = output_file + '_temp'
        with tf.python_io.TFRecordWriter(output_temp) as writer:

            for i, record_id in enumerate(record_ids):

                if i % 1000 == 0:
                    est_t = estimate_remaining_time(start_time, n_records, i)
                    logger.debug(
                        "Wrote %s / %s records (estimated time remaining: %s)"
                        % (i, n_records, est_t))

                record_data = self.tfrecord_dict[record_id]

                serialized_record = self._serialize_record(record_data)

                if serialized_record is None:
                    logger.debug("Discarding record %s - no image avail" %
                                 record_id)
                    continue

                # Write the serialized data to the TFRecords file.
                writer.write(serialized_record)
                successfull_writes += 1

        # Rename temporary file
        os.replace(output_temp, output_file)

        logger.info(
            "Finished Writing Records to %s - Wrote %s/%s" %
            (output_file, successfull_writes, n_records))

    def _write_to_file_parallel(self, output_file, record_ids):
        """ Write a TFR File with parallel image processing """

        # Create and Write Records to TFRecord file
        logger.info("Start Writing %s" % output_file)
        n_records = len(record_ids)
        successfull_writes = 0
        start_time = time.time()

        # Randomly shuffle records before saving, this is better for
        # model training
        if self.random_shuffle_before_save:
            random.seed(123)
            random.shuffle(record_ids)

        # divide writes into batches
        max_batch_size = min(self.process_images_in_parallel_size, n_records)
        n_batches = (n_records // max_batch_size) \
            + min(n_records % max_batch_size, 1)
        slices = slice_generator(n_records, n_batches)

        # temporary filename for writing to avoid complications after
        # a write is incomplete

        output_temp = output_file + '_temp'
        with tf.python_io.TFRecordWriter(output_temp) as writer:

            for batch_i, (start_i, end_i) in enumerate(slices):

                # Divide current batch to multiple processes
                record_batch = [self.tfrecord_dict[x] for x in
                                record_ids[start_i: end_i]]
                n_records_in_batch = len(record_batch)

                process_slices = slice_generator(
                    n_records_in_batch,
                    self.processes_images_in_parallel_n_processes)

                manager = Manager()
                serialized_records = manager.list()

                processes_list = list()

                for p_i, (batch_start_i, batch_end_i) in enumerate(process_slices):

                    process_batch = record_batch[batch_start_i: batch_end_i]
                    pr = Process(target=self._serialize_record_batch,
                                 args=(process_batch, serialized_records))
                    pr.start()
                    processes_list.append(pr)

                for p in processes_list:
                    p.join()

                # Write the serialized data to the TFRecords file.
                for serialized_record in serialized_records:
                    writer.write(serialized_record)
                    successfull_writes += 1
                est_t = estimate_remaining_time(start_time, n_records,
                                                successfull_writes)

                msg = "Wrote %s / %s records - \
                       estimated time remaining: %s - file: %s" % \
                      (successfull_writes, n_records, est_t, output_file)

                logger.debug(textwrap.shorten(msg, width=99))

        # Rename temporary file
        os.replace(output_temp, output_file)

        logger.info(
            "Finished Writing Records to %s - Wrote %s/%s" %
            (output_file, successfull_writes, n_records))


from __future__ import annotations
import glob
import shutil

import subprocess
import sys

import semver

def main(args):
    assert len(args) == 2
    check_cleanliness()
    new_version = bump_version(args[1])
    update_changelog(new_version)
    make_publication_commit(new_version)
    publish_to_pypi()
    prepare_next_version(new_version)

def check_cleanliness():
    not_indexed = subprocess.run(["git", "diff", "--stat", "--exit-code"])
    if not_indexed.returncode != 0:
        print("ERROR: there are non-staged changes. Please 'git add' them and retry.")
        exit(1)

    branch = subprocess.run(["git", "branch", "--show-current"], stdout=subprocess.PIPE, universal_newlines=True, check=True)
    if branch.stdout.strip() != "main":
        input("WARNING: you're not on branch 'main'. Press enter to proceed, Ctrl+C to cancel.")

    not_commited = subprocess.run(["git", "diff", "--stat", "--staged", "--exit-code"], stdout=subprocess.DEVNULL)
    if not_commited.returncode != 0:
        input("WARNING: Some changes are staged but not committed. They will be included in the publication commit. Press enter to proceed, Ctrl+C to cancel.")

def bump_version(part):
    with open("setup.py") as f:
        setup_lines = f.readlines()
    for line in setup_lines:
        if line.startswith("version = "):
            dev_version = semver.VersionInfo.parse(line[11:-2])

    assert dev_version.prerelease == "dev"
    assert dev_version.build is None

    print("Development version:", dev_version)
    if part == "patch":
        new_version = dev_version.replace(prerelease=None)
    elif part == "minor":
        new_version = dev_version.bump_minor()
    elif part == "major":
        new_version = dev_version.bump_major()
    else:
        assert False
    print("New version:", new_version)

    with open("setup.py", "w") as f:
        for line in setup_lines:
            if line.startswith("version = "):
                f.write(f"version = \"{new_version}\"\n")
            else:
                f.write(line)

    return new_version

def update_changelog(new_version):
    # @todo(later) Fix order of changelog: put recent versions first

    tags = subprocess.run(
        ["git", "tag"],
        stdout=subprocess.PIPE, universal_newlines=True,
        check=True,
    ).stdout.splitlines()
    last_tag = None
    for tag in tags:
        assert tag.startswith("v")
        if last_tag is None or semver.compare(tag[1:], last_tag[1:]) > 0:
            last_tag = tag

    log_lines = subprocess.run(
        ["git", "log", "--oneline", "--no-decorate", f"{last_tag}.."],
        stdout=subprocess.PIPE, universal_newlines=True,
        check=True,
    ).stdout.splitlines()

    with open("CHANGELOG.md", "a") as f:
        f.write(f"\n# Version {new_version}\n\n")
        for line in log_lines:
            f.write(f"- {line.split(' ', 1)[1]}\n")

    input("Please edit CHANGELOG.md then press enter to proceed, Ctrl+C to cancel.")

def make_publication_commit(new_version):
    subprocess.run(["git", "add", "setup.py", "CHANGELOG.md"], check=True)
    subprocess.run(["git", "commit", "-m", f"Publish version {new_version}"], stdout=subprocess.DEVNULL, check=True)
    subprocess.run(["git", "tag", f"v{new_version}"], check=True)
    subprocess.run(["git", "push", f"--tags"], check=True)

def publish_to_pypi():
    shutil.rmtree("dist", ignore_errors=True)
    shutil.rmtree("build", ignore_errors=True)
    shutil.rmtree("Chrones.egg-info", ignore_errors=True)
    subprocess.run(["python3", "-m", "build"])
    # The --repository option on next line assumes ~/.pypirc contains:
    # [distutils]
    #   index-servers=
    #     ...
    #     Chrones
    # [Chrones]
    #   repository = https://upload.pypi.org/legacy/
    #   username = __token__
    #   password = ... a token for package Chrones
    subprocess.run(["twine", "upload", "--repository", "Chrones"] + glob.glob("dist/*"), check=True)

def prepare_next_version(new_version):
    with open("setup.py") as f:
        setup_lines = f.readlines()

    next_version = new_version.bump_patch().replace(prerelease="dev")

    with open("setup.py", "w") as f:
        for line in setup_lines:
            if line.startswith("version = "):
                f.write(f"version = \"{next_version}\"\n")
            else:
                f.write(line)

    subprocess.run(["git", "add", "setup.py"], check=True)
    subprocess.run(["git", "commit", "-m", f"Start working on next version"], stdout=subprocess.DEVNULL, check=True)
    subprocess.run(["git", "push"], check=True)

if __name__ == "__main__":
    main(sys.argv)


# -*- coding: utf-8 -*-

class Tiempo:
    Hora = 0
    Minutos = 0

    def __init__(self, hora, mins):
        self.Hora = hora
        self.Minutos = mins

    def setHora(self, hora):
        self.Hora = hora

    def setMinutos(self, minutos):
        self.Minutos = minutos

    def getHora(self):
        if(self.Hora >= 0 and self.Hora < 24):
            return self.Hora
        else:
            return 0

    def getMinutos(self):
        if(self.Minutos >= 0 and self.Minutos < 60):
            return self.Minutos
        else:
            return 0

    def IntervaloTemporal(t1, t2):
        return [t1.getHora(),":",t1.getMinutos(),"-",t2.getHora(),":",t2.getMinutos()]

    def Pertenencia(t1, t2):
        p = "Pertenece a la "
        if(t1.getHora() < 6):
            p += "madrugada"
            if(t2.getHora() >= 6 and t2.getHora() < 12):
                p += " mañana"
            elif(t2.getHora() >= 12 and t2.getHora() < 18):
                p += " tarde"
            elif(t2.getHora() >= 18):
                p += " noche"
        elif(t1.getHora() >= 6 and t1.getHora() < 12):
            p += "mañana"
            if(t2.getHora() >= 12 and t2.getHora() < 18):
                p += " tarde"
            elif(t2.getHora() >= 18):
                p += " noche"
            elif(t2.getHora() < 6):
                p += " madrugada"
        elif(t1.getHora() >= 12 and t1.getHora() < 18):
            p += "tarde"
            if(t2.getHora() >= 18):
                p += " noche"
            elif(t2.getHora() < 6):
                p += " madrugada"
            elif(t2.getHora() >= 6 and t2.getHora() < 12):
                p += " mañana"
        elif(t1.getHora() >= 18):
            p += "noche"
            if(t2.getHora() < 6):
                p += " madrugada"
            elif(t2.getHora() >= 6 and t2.getHora() < 12):
                p += " mañana"
            elif(t2.getHora() >= 12 and t2.getHora() < 18):
                p += " tarde"
        return p

t1 = Tiempo(5, 23)
t2 = Tiempo(18,42)
print(Tiempo.IntervaloTemporal(t1, t2), Tiempo.Pertenencia(t1,t2))


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import requests as req

# In[2]:

res = req.get('https://www.denvergov.org/content/denvergov/en/city-of-denver-home/news.html')

# In[3]:

res.content

# Specify html parser to use, then import beautiful soup to clean up the content,

# In[4]:

get_ipython().run_line_magic('pinfo', 'bs')

# In[5]:

from bs4 import BeautifulSoup as bs
soup = bs(res.content, 'lxml')

# Now by utilizing beautiful soup, we have a clean version of our soup call as compared to the output of line 8

# In[6]:

soup

# Now we will move on to using the soup variable to obtain data from the website...  Navigating back to the Denver News website, we can right click on any of the links and click inspect.  This will allow us to view the html tags and code for the news articles.  Specifically we are looking for the date along with the associate title. Xpaths can be brittle and change frequently; so for this exercise we will be looking for <div class "Denver-list-item">
#
# This is accomplished by utilizing the soup object, div being our first argument, and our second argument being a dictionary telling the class to be denver news list item
#

# In[7]:

divs = soup.find_all('div', {'class': 'denver-news-list-item'})

# In[8]:

divs[0]

# Getting the numeber of articles on this page:

# In[9]:

len(divs)

# We can see that these are beautiful soup tag objects, as shown below

# In[10]:

type(divs[0])

# With this we can view the text inside of it

# In[11]:

divs[0].text

# We can get certain components as shown below:

# In[13]:

divs[0].get('href')

# Looking at the element below we can see the date is noted in the second paragrahp tag with a class showing us dates.   A clean layout which is not typical of most websites

# In[14]:

divs[0]

# In[15]:

divs[0].find('p', {'class': 'denver-news-list-date'})

# In[16]:

import pandas as pd

# In[17]:

pd.to_datetime(divs[0].find('p', {'class': 'denver-news-list-date'}).text)

# In[18]:

BASE_URL = 'https://www.denvergov.org'

# In[19]:

divs[0].find('a')

# In[20]:

divs[0].find('a').text #giving us the title

# In[21]:

divs[0].find('a').get('href')

# In[22]:

BASE_URL + divs[0].find('a').get('href')

# Now we have all the different elements of the web page, so now we just need to loop through each of the divs

# In[23]:

for d in divs:
    #get date
    date = pd.to_datetime(d.find('p', {'class': 'denver-news-list-date'}).text)
    #get link
    link = d.find('a')
    #get href component
    href = BASE_URL + link.get('href')
    #get title
    title = link.text

# In[24]:

title

# In[25]:

date

# In[26]:

href

# The next thing to address is the number of pages, the url will change with the page number. So what we want to do is loop through all the pages, and this is accomplished by utilizing the curly brackets to incorporate all pages as shown below.  The call will fill in the curly brackets.  Format(2) identifies the page that we want.  Below we get a nice view of the url pattern.

# In[27]:

for i in range(1,10):
    print('https://www.denvergov.org/content/denvergov/en/city-of-denver-home/news.html?page={}'.format(2))

# The last step is to find the last page, you can refernce the results x of x number.  We can accomplish this by setting the page in the url to a large one, 100 works for this example.  In my results p 29 was the last one. So we can rerun the loop with the updated parameters:

# In[106]:

list(range(1,29)) #note it only goes from 1:28

# In[107]:

#to get to page 29 set range from 1:30, giving us all pages from the website
for i in range(1,30):
    print('https://www.denvergov.org/content/denvergov/en/city-of-denver-home/news.html?page={}'.format(i))

# Now we can combine all the elements into one consise loop as shown below:

# In[108]:

for i in range(1,30):
    url = ('https://www.denvergov.org/content/denvergov/en/city-of-denver-home/news.html?page={}'.format(i))
    res = req.get(url)

# In[109]:

from pymongo import MongoClient
from tqdm import tqdm_notebook #getting a status bar

PAGE_URL = 'https://www.denvergov.org/content/denvergov/en/city-of-denver-home/news.html?page={}'

#Store in MongoDB
client = MongoClient()
#create db
db = client['news']
#set up collection
coll = db['denver']

for page in tqdm_notebook(range(1,30)):
    if page==3:
        break
    url = PAGE_URL.format(page)
    res = req.get(url)
    soup = bs(res.content)
    divs = soup.find_all('div', {'class': 'denver-news-list-item'})

    for d in divs:
        date = pd.to_datetime(d.find('p', {'class': 'denver-news-list-date'}).text)
        link = d.find('a')
        href = BASE_URL + link.get('href')
        title = link.text

        #giving keys and values using dictionary
        coll.insert_one({'date': date,
                        'link': href,
                        'title': title})

#Close connection MogoDb
client.close()

# In[110]:

from pymongo import MongoClient
from tqdm import tqdm_notebook #getting a status bar

PAGE_URL = 'https://www.denvergov.org/content/denvergov/en/city-of-denver-home/news.html?page={}'

#Store in MongoDB
client = MongoClient()
#create db
db = client['news']
#set up collection
coll = db['denver']

for page in tqdm_notebook(range(1,30)):
    #if page==3:
      #  break
    url = PAGE_URL.format(page)
    res = req.get(url)
    soup = bs(res.content)
    divs = soup.find_all('div', {'class': 'denver-news-list-item'})

    for d in divs:
        date = pd.to_datetime(d.find('p', {'class': 'denver-news-list-date'}).text)
        link = d.find('a')
        href = BASE_URL + link.get('href')
        title = link.text

        #giving keys and values using dictionary
        coll.insert_one({'date': date,
                        'link': href,
                        'title': title})

#Close connection MogoDb
client.close()

# In[111]:

from pprint import pprint

client = MongoClient()
db = client['news']
coll = db['denver']

pprint(coll.find_one())

# For my web scraping deliverable I decided that I would scrape the colorado news at https://www.colorado.gov/news

# In[28]:

res2 = req.get('https://www.colorado.gov/news')

# In[29]:

res2.content

# In[30]:

get_ipython().run_line_magic('pinfo', 'bs')
from bs4 import BeautifulSoup as bs
soup2 = bs(res2.content)

# In[31]:

soup2

# Navigating back to the webpage we're looking for the div class tags for each article as shown below:

# In[32]:

divs2 = soup2.find_all('div', {'class': 'view-page-item'})

# In[33]:

divs2[0]

# In[34]:

#Number of articles
len(divs2)

# In[35]:

divs2[0].text

# In[36]:

import pandas as pd
divs2[0].find('span', {'class': 'date-display-single'}).text

# In[37]:

pd.to_datetime(divs2[0].find('span', {'class': 'date-display-single'}).text)

# In[38]:

BASE_URL2 = 'https://www.colorado.gov/news'

# In[39]:

divs2[0].find('a').text #title

# In[40]:

divs2[0].find('a').get('href')

# In[41]:

divs2[0].find('div', {'class': 'smaller-text-views'}).text.strip()

# In[42]:

BASE_URL2 + divs2[0].find('a').get('href')

# Now we have all the elements, so we will just need to loop through as shown below:

# In[128]:

for d in divs2:
    link = d.find('a')
    date = pd.to_datetime(divs2[0].find('span', {'class': 'date-display-single'}).text)
    href = BASE_URL2 + link.get('href')
    title = link.text

# In[43]:

title

# In[44]:

link

# In[45]:

date

# In[46]:

href

# In[47]:

link

# In[135]:

for i in range(1,71): #70 is the last page with data within the colorado news website :
    print('https://www.colorado.gov/news?page={}'.format(i))

# In[ ]:

for i in range(1,71):
    url2 = ('https://www.colorado.gov/news?page?page={}'.format(i))
    res2 = req.get(url)

# In[50]:

import pandas as pd
from tqdm import tqdm_notebook
from pymongo import MongoClient

PAGE_URL2 = 'https://www.colorado.gov/news?page?page={}'

client = MongoClient()
db = client['news2']
coll = db['Colorado']

for page in tqdm_notebook(range(0, 71)):
#     for testing, we can stop it after a few iterations
#     if page == 3:
#         break
    res2 = req.get(PAGE_URL2.format(page))
    soup2 = bs(res2.content, 'lxml')
    divs2 = soup2.find_all('div', {'class': 'views-row'})
    for d in divs2:
        date = pd.to_datetime(d.find('span', {'class': 'date-display-single'}).text)
        link = divs2[0].find('a')
        title = link.text
        href = link.get('href')
        subtitle = d.find('div', {'class': 'smaller-text-views'}).text.strip()

        coll.insert_one({'date': date,
                         'title': title,
                         'link': href,
                         'subtitle': subtitle})

client.close()

# In[51]:

from pprint import pprint

client = MongoClient()
db = client['news2']
coll = db['Colorado']

pprint(coll.find_one())

MSDS_610_X40 Data Engineering - Week 6 Web Scraping .py


import numpy as np
import sys
sys.path.insert(0,"..")
from evaluation.EvalRSRunner import ChallengeDataset
import pandas as pd
from reclist.abstractions import RecModel

from sklearn.decomposition import TruncatedSVD

import time
import random

from tqdm import tqdm

# from sklearn.preprocessing import OneHotEncoder
from sklearn.decomposition import NMF

from sklearn.metrics.pairwise import cosine_similarity

from scipy.sparse import csr_matrix

class MyModel(RecModel):

    def __init__(self, tracks: pd.DataFrame, users: pd.DataFrame, top_k : int = 100, **kwargs):
        self.top_k = top_k

    def train(self, train_df: pd.DataFrame):
        self.user_map = { k: v for v, k in enumerate(list(set(train_df["user_id"]))) }
        self.rev_user_map = { k: v for v,k in self.user_map.items() }
        self.track_map = { k: v for v, k in enumerate(list(set(train_df["track_id"]))) }
        self.rev_track_map = { k: v for v,k in self.track_map.items() }

        t1 = time.time()
        data = []
        row_ind = []
        col_ind = []

        self.train_df = train_df

        for _, row in train_df.iterrows():
            data.append(np.log(1+row.user_track_count)) # use num_plays instead?
            row_ind.append(self.user_map[row.user_id])
            col_ind.append(self.track_map[row.track_id])

        self.mat = csr_matrix((data, (row_ind, col_ind)), shape=(len(self.user_map), len(self.track_map)))
        t2 = time.time()
        print("Matrix generated in ", round(t2-t1,2), "seconds. Shape:", self.mat.shape)

        svd = TruncatedSVD(128)

        Us = svd.fit_transform(self.mat)
        t3 = time.time()
        print("SVD done in", round(t3-t2,2), "seconds")
        V = svd.components_

        self.Us = Us
        self.V = V

        # decomp = NMF(n_components=2, init="random")

    def predict(self, user_ids: pd.DataFrame) -> pd.DataFrame:

        self.users_emb = self.Us.astype(np.float32)
        self.tracks_emb = self.V.T.astype(np.float32)

        print("Computing similarities")
        start = time.time()
        # cos_mat = cossim(users_emb, tracks_emb).cpu().detach().numpy()
        cos_mat = cosine_similarity(self.users_emb, self.tracks_emb)
        # cos_mat = self.users_emb @ self.track_emb
        stop = time.time()
        print("Similarities computed in", stop-start, "seconds")
        self.cos_mat = cos_mat

        known_tracks_array = np.array([ self.rev_track_map[i] for i in range(len(self.track_map))])
        results = np.zeros((len(user_ids), self.top_k), dtype=int)

        known_likes = {}
        for user, grp in self.train_df.groupby("user_id"):
            known_likes[user] = set(grp["track_id"])

        overlaps = []
        horizon = 1
        with tqdm(range(cos_mat.shape[0])) as bar:
            for i in bar:
                curr_k = self.top_k * horizon + len(known_likes[int(user_ids.iloc[i])])

                parts = np.argpartition(-cos_mat[i], kth=curr_k)[:curr_k]
                cos_mat_sub = known_tracks_array[parts[np.argsort(-cos_mat[i,parts])]]
                chosen = np.zeros(self.top_k * horizon)
                j = 0
                k = 0
                overlaps.append(len(set(cos_mat_sub)&known_likes[int(user_ids.iloc[i])]) / len(known_likes[int(user_ids.iloc[i])]))
                ### TODO: overlap between found and known is small!!!
                while k < self.top_k * horizon:
                    if cos_mat_sub[j] not in known_likes[int(user_ids.iloc[i])]:
                        chosen[k] = cos_mat_sub[j]
                        k += 1
                    j += 1

                p = 1/(1+np.arange(len(chosen)))# + 1 * tracks_pop[chosen].values + 1 * tracks_artist_pop[chosen].values

                p = p / p.sum()
                subset = np.random.choice(len(chosen), size=self.top_k, p=p, replace=False)

                results[i] = chosen[sorted(subset)] #sort_by_order(chosen, subset)

        preds = results
        data = np.hstack([ user_ids["user_id"].values.reshape(-1, 1), preds ])
        return pd.DataFrame(data, columns=['user_id', *[str(i) for i in range(self.top_k)]]).set_index('user_id')

# if __name__ == "__main__":

#     dataset = ChallengeDataset()
#     train, test = dataset.get_sample_train_test()

#     model = MyModel(dataset.df_tracks, dataset.df_users)
#     model.train(train)
#     model.predict(test[["user_id"]])


#crie um programa q leia ano de nascimento de 7 pessoas
#no final, mostre qnts pessoas ainda não atingiram a maioridade e
#qnts são maiores de idade (21 anos)

from datetime import date

maiores = 0
menores = 0
print('SER ADULTO É UMA DROGA')
print('Será que você já é adulto?')
for c in range(1, 8):
    ano = int(input('Digite seu ano de nascimento: '))
    idade = date.today().year - ano
    if idade <= 21:
        maiores += 1
    else:
        menores += 1
print('Nessa lista, {} pessoas são maiores de idade e {} são menores de idade.'.format(maiores, menores))


import plotly.figure_factory as ff
import pandas
import statistics
# read csv
df= pandas.read_csv("data.csv")

# finding mean,median,mode of the height
height=df["Height(Inches)"].tolist()
mean=statistics.mean(height)
mode=statistics.mode(height)
median=statistics.median(height)

# calculating std _deviation
std_deviation=statistics.stdev(height)

#printing mean.median.mode.std_deviation
print("Mean of height of students:",mean)
print("Median of height of students:",median)
print("Mode of height of students:",mode)
print("Standard Deviation of height of students:",std_deviation)
# plot the graph
#figure=ff.create_distplot([height],["height of students"])
#figure.show()
# finding the first std deviation
upper_limit=mean+std_deviation
lower_limit=mean-std_deviation
print (upper_limit,lower_limit)
#run a loop on height if the height lies betwween upper_limit and lower_limit
list1=[]
for i in height:
    if lower_limit<i<upper_limit:
        list1.append(i)

per1=(len(list1)/len(height))*100
print("percentage of data lying in 1 st std deviation:", per1)

# finding 2 nd std deviation
upper_limit=mean+2*std_deviation
lower_limit=mean-2*std_deviation
print (upper_limit,lower_limit)
#run a loop on height if the height lies betwween upper_limit and lower_limit
list2=[]
for i in height:
    if lower_limit<i<upper_limit:
        list2.append(i)

per2=(len(list2)/len(height))*100
print("percentage of data lying in 2st std deviation:", per2)

# finding 3 nd std deviation
upper_limit=mean+3*std_deviation
lower_limit=mean-3*std_deviation
print (upper_limit,lower_limit)
#run a loop on height if the height lies betwween upper_limit and lower_limit
list3=[]
for i in height:
    if lower_limit<i<upper_limit:
        list3.append(i)

per3=(len(list3)/len(height))*100
print("percentage of data lying in 3 rd std deviation:", per3)


# -*- coding: utf-8 -*-
"""
Created on Fri Aug  9 22:26:32 2019

@author: Akash
"""

import math
import os
import random
import re
import sys
def cal_req_pos(number,req_pos):
    req_pos.clear()
    for i in range(len(number)):
        req_pos.append(number[i]-1)
        print("req_pos",req_pos)
# Complete the minimumBribes function below.
def minimumBribes(a):

    numbers=[]
    real_pos=[]
    Bribes=[]

    for i in range(len(a)):
        if a[i]!=i+1:
            numbers.append(a[i])
            real_pos.append(i+1)

    print("real_pos",real_pos)

    print(numbers)
    req_pos=[]

    for i,j in zip(real_pos,req_pos):
        if abs(i-j)>2:
            print("too chotic")
            return
    for i,j in zip(real_pos,req_pos):
        if abs(i-j)==1:
            print(i,j)

#            a[i-1]=numbers[j-1]

    for i in range(len(numbers)):
        index=int(req_pos[i-1])-1
        num=int(numbers[i])
        a[index]=num
        cal_req_pos(numbers,req_pos)
#        a.insert(req_pos[i]-1,numbers[i])

    return a

if __name__ == '__main__':
    q=[1,2,3,4,5,8,6,7]

#    q[0]=q[3]
#    print(q)
    print(minimumBribes(q))


import os
import sys
import random

try:
    from hashlib import md5
except ImportError:
    from md5 import md5

import tornado.httpserver
import tornado.ioloop
import tornado.web

import time

import battlenet
from battlenet import Realm

from threading import Thread, Lock

realms = {}
myLock = Lock()

def synchronized(lock):
    """ WHEE! """
    def wrap(f):
    	def new_function(*args, **kw):
    	    lock.acquire()
    	    try:
    	        return f(*args, **kw)
    	    finally:
                lock.release()
    	return new_function
    return wrap

class MyThread(Thread):
    def run(self):
        self.populate_realms()
        time.sleep(180)
        self.run()

    @synchronized(myLock)
    def populate_realms(self):
        connection = battlenet.Connection()
        realmsData = connection.get_all_realms(battlenet.UNITED_STATES)
        for realm in realmsData:
            realms[realm.slug] = realm

class MainHandler(tornado.web.RequestHandler):
    def get(self, realm_slug = None):
        now = time.time()
        (realm_slug, realm) = select_realm(realm_slug)
        if realm_slug == None:
            raise tornado.web.HTTPError(404)
        status = "down"
        if realm.status == True:
            status = "up"
        message = realm.name + " is " + status
        self.output_message(message, realm_slug)

    def output_message(self, message, realm_slug):
        self.render('index.html', message = message, realm_slug = realm_slug)

class PlainTextHandler(MainHandler):
    def output_message(self, message, realm_slug):
        self.set_header('Content-Type', 'text/plain')
        self.write(message)

@synchronized(myLock)
def select_realm(realm_slug = None):
    if not realm_slug:
        realm_slug = random.choice(realms.keys())
    elif realm_slug not in realms:
        return (None, None)
    return (realm_slug, realms[realm_slug])

settings = {
    'static_path': os.path.join(os.path.dirname(__file__), 'static'),
}

application = tornado.web.Application([
    (r'/', MainHandler),
    (r'/([a-z0-9\-]+)', MainHandler),
    (r'/index.txt', PlainTextHandler),
    (r'/([a-z0-9\-]+)/index.txt', PlainTextHandler),
], **settings)

if __name__ == '__main__':
    port = 8000
    if len(sys.argv) > 1:
        port = int(sys.argv[1])

    thread = MyThread()
    thread.daemon = True
    thread.start()

    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(port)
    tornado.ioloop.IOLoop.instance().start()


def TransformationInput(data):
    data = str(data)
    if ":" in data:
        sData = data.split(':')

        result = sData[-1].split('.')
        result = int(result[0]) * 1000 + int(result[1])

        if len(sData) == 2:
            result += int(sData[0]) * 60 * 1000 # Преобразование минут в миллисекунды
        elif len(sData) == 3:
            result += (int(sData[0]) * 60 * 60 * 1000) + (int(sData[1]) * 60 * 1000)
        return result

    elif "," in data:
        return int(100 * float(data.replace(',', '.')))
    else:
        return int(data)

def TransformationOutput(data, dataType):
    if dataType in ['количество', 'сантиметры']:
        return data
    elif dataType == 'метры':
        return int(data)/100
    elif dataType in ['время', 'часы', 'минуты', 'секунды']:
        hour = int(data / (1000*60*60))
        data = data - (hour * 1000*60*60)
        hour = '0' + str(hour) if int(hour) < 10 else str(hour)

        minute = int(data / (1000*60))
        data = data - (minute * 1000*60)
        minute = '0' + str(minute) if int(minute) < 10 else str(minute)

        second = int(data / (1000))
        data = data - (second * 1000)
        second = '0' + str(second) if int(second) < 10 else str(second)

        msecond = ('0' * (3 - len(str(data)))) + str(data)

        return hour + ':' + minute + ":"  + second + '.' + msecond


import os
import glob
import platform
from conans import ConanFile, AutoToolsBuildEnvironment, tools

class ConanFileBase(ConanFile):
    _base_name = "flex"
    version = "2.6.4"
    url = "https://github.com/bincrafters/conan-flex"
    homepage = "https://github.com/westes/flex"
    description = "Flex, the fast lexical analyzer generator"
    topics = ("conan", "flex", "lex", "lexer", "lexical analyzer generator")
    license = "BSD-2-Clause"
    exports = ["LICENSE.md"]
    exports_sources = ["patches/*.patch"]
    requires = ("m4_installer/1.4.18@bincrafters/stable",)

    _source_subfolder = "source_subfolder"

    def build_requirements(self):
        if tools.os_info.is_windows:
            if "CONAN_BASH_PATH" not in os.environ:
                self.build_requires("msys2/20190524")

    def source(self):
        sha256 = "e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995"
        tools.get("{0}/releases/download/v{1}/{2}-{3}.tar.gz".format(self.homepage, self.version, self._base_name, self.version), sha256=sha256)
        extracted_dir = self._base_name + "-" + self.version
        os.rename(extracted_dir, self._source_subfolder)

    def configure(self):
        del self.settings.compiler.libcxx
        del self.settings.compiler.cppstd

    def requirements(self):
        if tools.os_info.is_windows:
            self.requires("pcre2/10.33")

    def _apply_patches(self):
        for filename in sorted(glob.glob("patches/*.patch")):
            self.output.info('applying patch "%s"' % filename)
            tools.patch(base_path=self._source_subfolder, patch_file=filename)

    @staticmethod
    def detected_os():
        if tools.os_info.is_macos:
            return "Macos"
        if tools.os_info.is_windows:
            return "Windows"
        return platform.system()

    @property
    def _the_os(self):
        return self.settings.get_safe("os") or self.settings.get_safe("os_build")

    @property
    def _the_arch(self):
        return self.settings.get_safe("arch") or self.settings.get_safe("arch_build")

    @property
    def cross_building(self):
        if tools.cross_building(self.settings):
            if self._the_os == self.detected_os():
                if self._the_arch == "x86" and tools.detected_architecture() == "x86_64":
                    return False
            return True
        return False

    def build(self):
        if tools.os_info.is_windows:
            self._apply_patches()
        env_build = AutoToolsBuildEnvironment(self, win_bash=tools.os_info.is_windows)
        configure_args = ["--disable-nls", "HELP2MAN=/bin/true", "M4=m4"]
        if "shared" in self.options and self.options.shared:
            configure_args.extend(["--enable-shared", "--disable-static"])
        else:
            configure_args.extend(["--disable-shared", "--enable-static"])

        with tools.chdir(self._source_subfolder):
            if self._the_os == "Windows":
                tools.save("regex.h", "#define PCRE2_CODE_UNIT_WIDTH 8\n#include <pcre2posix.h>")
                if not os.path.isdir("sys"):
                    os.makedirs("sys")
                tools.download("https://raw.githubusercontent.com/win32ports/sys_wait_h/master/sys/wait.h", os.path.join("sys", "wait.h"))
                env_build.include_paths.append(os.getcwd())
            if self._the_os == "Linux":
                # https://github.com/westes/flex/issues/247
                configure_args.extend(["ac_cv_func_malloc_0_nonnull=yes", "ac_cv_func_realloc_0_nonnull=yes"])
                # https://github.com/easybuilders/easybuild-easyconfigs/pull/5792
                configure_args.append("ac_cv_func_reallocarray=no")
            if self.cross_building:
                # stage1flex must be built on native arch: https://github.com/westes/flex/issues/78
                self.run("./configure %s" % " ".join(configure_args))
                env_build.make(args=["-C", "src", "stage1flex"])
                self.run("mv src/stage1flex src/stage1flex.build")
                env_build.make(args=["distclean"])
                with tools.environment_append(env_build.vars):
                    env_build.configure(args=configure_args)
                    cpu_count_option = "-j%s" % tools.cpu_count()
                    self.run("make -C src %s || true" % cpu_count_option)
                    self.run("mv src/stage1flex.build src/stage1flex")
                    self.run("touch src/stage1flex")
                    env_build.make(args=["-C", "src"])
            else:
                with tools.environment_append(env_build.vars):
                    env_build.configure(args=configure_args)
                    env_build.make()
            env_build.make(args=["install"])

    def package(self):
        self.copy(pattern="COPYING", dst="licenses", src=self._source_subfolder)


from __tools import *

def gauss_seidel(matrix_A, vector_b, x0, TOL, MAX_ITER=100):
    n = len(matrix_A)
    x = np.copy(x0)
    x_new = np.zeros_like(x).astype(np.float64)

    for k in range(MAX_ITER):
        aux = np.copy(x)
        for i in range(n):
            s = sum(matrix_A[i, j] * x[j] for j in range(n) if j != i)
            x_new[i] = (vector_b[i] - s) / matrix_A[i, i]
            x[i] = x_new[i]

        if np.linalg.norm((aux - x_new) / x_new) < TOL:
            return x_new

        x = x_new

    print('> O método de Gauss Seidel atingiu o número máximo de iterações.')
    return x

def run():
    results = []
    x0 = [0., 0., 0.]
    mx_A, mx_b, TOL, MAX_ITER = read_system('input.txt')

    results.append({'name': 'gauss_seidel', 'result': gauss_seidel(mx_A, mx_b, x0, TOL, MAX_ITER)})
    save_results('output.txt', results)

# Chama a função principal
if __name__ == '__main__':
    run()


from django.urls import path
from rest_framework_simplejwt.views import (
    TokenObtainPairView,
    TokenRefreshView,
)
from .views import SocialAuthenticationView

from . import views

app_name = "authentication"
urlpatterns = [

    path('login', views.SigninView.as_view(), name='token_obtain_pair'),
    path('auth/token/refresh', TokenRefreshView.as_view(), name='token_refresh'),
    path('auth/registration',
          views.SingleUserRegistrationView.as_view(),name='SingleUserRegistration'),
    path('auth/logout/', views.SignoutView.as_view(), name="SignoutView"),

    path('login/oauth',
          views.SocialAuthenticationView.as_view(),name='social_auth')

]


import couchdb,os

# function that creartes a connection to couchdb
def create_connection(url,dbName=None):
    # create the connection to couchdb
    couch = couchdb.Server(url)
    # create the database
    if dbName == None:
        return couch
    else:
        db = couch[dbName]
        return couch,db

# function that creates dummy db and returns couch,db,df
def create_dummy_db(url,dbName):
    # create the connection to couchdb
    couch = create_connection(url)
    # create dummy dataframe
    import pandas as pd
    df = pd.DataFrame(
        {
        "id_no": [1,2,3,4,5,6,7,8,9,10],
        "ProjectTitle": ["Project1","Project2","Project3","Project4","Project5","Project6","Project7","Project8","Project9","Project10"],
        "ReportNumber": ["Report1","Report2","Report3","Report4","Report5","Report6","Report7","Report8","Report9","Report10"],
        "ToName": ["To1","To2","To3","To4","To5","To6","To7","To8","To9","To10"],
        "ProjectDetails": ["ProjectDetails1","ProjectDetails2","ProjectDetails3","ProjectDetails4","ProjectDetails5","ProjectDetails6","ProjectDetails7","ProjectDetails8","ProjectDetails9","ProjectDetails10"],
        "status": ["status1","status2","status3","status4","status5","status6","status7","status8","status9","status10"],
        "Date": ["01-01-2020","02-02-2020","03-03-2020","04-04-2020","05-05-2020","06-06-2020","07-07-2020","08-08-2020","09-09-2020","10-10-2020"],
        "Message": ["Message1","Message2","Message3","Message4","Message5","Message6","Message7","Message8","Message9","Message10"],
        "Incharge": ["Incharge1","Incharge2","Incharge3","Incharge4","Incharge5","Incharge6","Incharge7","Incharge8","Incharge9","Incharge10"],
         "clientMail": ["a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com","a@gmail.com"]
    }
    )
    df.set_index('id_no', inplace=True)
    # create the database
    db = couch.create(dbName)
    # save the dataframe to database
    for index,row in df.iterrows():
        tmpDict = row.to_dict()
        tmpDict['id_no'] = index
        # print(tmpDict)
        db.save(tmpDict)
    return couch,db,df

# function that returns the dataframe of the database
def get_dataframe_from_db(db):
    # read all records of database
    records = [db[id] for id in db]
    # convert to pandas dataframe
    import pandas as pd
    dbDf= pd.DataFrame(records)
    dbDf['id_no'] = dbDf['id_no'].astype(int)
    return dbDf

# function that returns df of the database <=url,dbName => dbDf
def getDbDf(url,dbName):
    # create the connection to couchdb
    couch,db = create_connection(url,dbName)
    # read all records of database
    records = [db[id] for id in db]
    # convert to pandas dataframe
    import pandas as pd
    dbDf= pd.DataFrame(records)
    dbDf['id_no'] = dbDf['id_no'].astype(int)
    dbDf.set_index('id_no', inplace=True)
    return dbDf

#
#  def save_email(db,df):
#     # create the email
#     email = {
#         "subject": "Status Update",
#         "body": "Hi,\n\nYour status has been updated.\n\nRegards,\n\nTeam",
#         "to": df['clientMail'],
#         "from": os.environ["MAIL_USERNAME"],
#         "attachments": [
#             {
#                 "name": "status.txt",
#                 "content": df['status']
#             }
#         ]
#     }
#     # save the email to draft

# now lets read


#!/user/bin/env python3
# -*- coding: utf-8 -*-

# Python是动态语言，根据类创建的实例可以任意绑定属性。

class Student(object):
    # 这个属性虽然归类所有，但类的所有实例都可以访问到
    age = 15
    def __init__(self, name):
        self.name = name

s = Student('Bob')
s.score = 90
print(s.score)
# 输出：90

s.age = 30
print(s.age)
# 输出：30
print(Student.age)
# 输出：15

# 删除实例的属性
del s.age
print(s.age)     # 显示类的属性
# 输出：15

# 在编写程序的时候，千万不要对实例属性和类属性使用相同的名字，因为相同名称的实例属性将屏蔽掉类属性，
# 但是当你删除实例属性后，再使用相同的名称，访问到的将是类属性。

# 实例属性属于各个实例所有，互不干扰；
# 类属性属于类所有，所有实例共享一个属性；
# 不要对实例属性和类属性使用相同的名字，否则将产生难以发现的错误。

# 为了统计学生人数，可以给Person类增加一个类属性，每创建一个实例，该属性自动增加
class Person(object):
    count = 0
    def __init__(self, name):
        self.name = name
        Person.count += 1

if Person.count != 0:
    print('Failed!')
else:
    bart = Person('Bart')
    if Person.count != 1:
        print('Failed!')
    else:
        bob = Person('Bob')
        if Person.count != 2:
            print('Failed!')
        else:
            print('Person:', Person.count)
            print('Success!')
# 输出：Person: 2
#      Success!


def ceaserEncrypt(str, rot):
    str = str.lower()
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    strNew = ''
    for c in str:
        if c in alphabet:
            strNew += alphabet[(alphabet.index(c)+int(rot)) % len(alphabet)]
        else:
            strNew += c
    return strNew

def ceaserDecrypt(str, rot):
    str = str.lower()
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    strOrg = ''
    for c in str:
        if c in alphabet:
            strOrg += alphabet[(alphabet.index(c)-int(rot)) % len(alphabet)]
        else:
            strOrg += c
    return strOrg

def ceaserCrack(str):

    max = 0
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    weight = [6.51, 1.89, 3.06, 5.08, 17.4,
              1.66, 3.01, 4.76, 7.55, 0.27,
              1.21, 3.44, 2.53, 9.78, 2.51,
              0.29, 0.02, 7.00, 7.27, 6.15,
              4.35, 0.67, 1.89, 0.03, 0.04, 1.13]

    countLetters = []
    sLetters = []
    for i in range(0, len(alphabet)):
        countLetters.append(0)
        sLetters.append(0)

    for c in str:
        if c in alphabet:
            countLetters[alphabet.index(c)] += 1

    for i in range(0, len(alphabet)):
        for j in range(0, len(alphabet)):
            sLetters[i] += 0.01 * countLetters[j] * weight[(j + i) % 26]
            if max < sLetters[i]:
                max = sLetters[i]

    return (26 - sLetters.index(max)) % 26


INF = int(1e9)  # 무한을 의미하는 INF 값을 10억으로 설정

# 노드의 개수, 간선의 개수 입력
n, m = map(int, input().split())

# 2차원 리스트(그래프)를 만들고, 모든 값을 무한으로 초기화
graph = [[INF] * (n + 1) for _ in range(n + 1)]

# 자기 자신에서 자기 자신으로 가는 비용은 0으로 초기화
for a in range(n + 1):
    for b in range(n + 1):
        if a == b:
            graph[a][b] = 0

# 각 간선에 대한 정보를 입력 받아, 그 값으로 초기화
for _ in range(m):
    # A와 B가 서로에게 가는 비용 1
    a, b = map(int, input().split())
    graph[a][b] = 1
    graph[b][a] = 1

# 거쳐 갈 노드 X와 최종 목적지 K 입력
x, k = map(int, input().split())

# 플로이드 워셜 알고리즘 수행
for k in range(1, n + 1):
    for a in range(1, n + 1):
        for b in range(1, n + 1):
            graph[a][b] = min(graph[a][b], graph[a][k] + graph[k][b])

# 수행 결과 출력
distance = graph[1][k] + graph[k][x]

# 도달할 수 없으면 -1, 있으면 최단 거리 출력
if distance >= INF:
    print("-1")
else:
    print(distance)


import subprocess
import os

path =  os.path.dirname(os.path.abspath(__file__))
auxdata = os.path.join(path, "../auxdata")

def s2_preprocessing(gpt_path:str, S2_L2A:str, AOI:str,
    out_refl:str, out_sun_zenith:str, out_mask:str, out_bio:str):
    """Running pre-processing procedures for Sentinel 2 data.

    Args:
        gpt_path (str): Path to SNAP GPT
        S2_L2A (str): Path to Sentinel 2 L2A data
        AOI (str): WKT geometry as string
        out_refl (str): Path to store the output reflectance product
        out_sun_zenith (str): Path to store sun zenith angles product
        out_mask (str): Path to store the output mask product
        out_bio (str): Path to store the output bio product
    """

    subprocess.run([gpt_path, os.path.join(auxdata, "sentinel_2_preprocessing.xml"),
        "-PINPUT_S2_L2A={}".format(S2_L2A),
        "-PAOI={}".format(AOI),
        "-POUTPUT_REFL={}".format(out_refl),
        "-POUTPUT_SUN_ZEN_ANG={}".format(out_sun_zenith),
        "-POUTPUT_MASK={}".format(out_mask),
        "-POUTPUT_BIO={}".format(out_bio)])

def elevation(gpt_path:str, in_mask:str, out_elev:str):
    """Running SNAP GPT graph to extract elevation data.

    Args:
        gpt_path (str): Path to SNAP GPT
        in_mask (str): Path to input mask (out_refl from s2_preprocessing)
        out_elev (str): Path to store the elevation data
    """

    subprocess.run([gpt_path, os.path.join(auxdata, "add_elevation.xml"),
        "-PINPUT_S2_MASK={}".format(in_mask),
        "-POUTPUT_SRTM_ELEV={}".format(out_elev)
        ])

def landcover(gpt_path:str, in_mask:str, out_lc:str):
    """Running SNAP GPT graph to extract LandCover data.

    Args:
        gpt_path (str): Path to SNAP GPT
        in_mask (str): Path to input mask (out_refl from s2_preprocessing)
        out_lc (str): Path to store the LandCover data
    """

    subprocess.run([gpt_path, os.path.join(auxdata, "add_landcover.xml"),
        "-PINPUT_S2_MASK={}".format(in_mask),
        "-POUTPUT_CCI_LC={}".format(out_lc)
        ])

def s3_preprocessing(gpt_path:str, S3_L2:str, AOI:str, out_obs_geom:str,
    out_mask:str, out_lst:str):
    """Preprocessing procedures for Sentinel 3 LST data.

    Args:
        gpt_path (str): Path to SNAP GPT
        S3_L2 (str): Path to Sentinel 3 image
        AOI (str): WKT geometry as string
        out_obs_geom (str): Path to store observation geometry file
        out_mask (str): Path to store mask file
        out_lst (str): Path to store LST file
    """

    subprocess.run([gpt_path, os.path.join(auxdata, "sentinel_3_preprocessing.xml"),
        "-PINPUT_S3_L2={}".format(S3_L2),
        "-PINPUT_AOI_WKT={}".format(AOI),
        "-POUTPUT_observation_geometry={}".format(out_obs_geom),
        "-POUTPUT_mask={}".format(out_mask),
        "-POUTPUT_LST={}".format(out_lst)])


import copy

A = [list(input()) for i in range(10)]
A_copy = copy.deepcopy(A)

land_count = 0
reached = [[0] * 10 for i in range(10)]

def landfill(x, y):
    if x < 0 or 10 <= x or y < 0 or 10 <= y or A_copy[x][y] == "x":
        return
    if reached[x][y] == 1:
        return

    reached[x][y] = 1

    landfill(x+1, y)
    landfill(x-1, y)
    landfill(x, y+1)
    landfill(x, y-1)

for i in range(10):
    for j in range(10):
        if A[i][j] == "o":
            land_count += 1

for i in range(10):
    for j in range(10):
        reached = [[0] * 10 for i in range(10)]
        A_copy = copy.deepcopy(A)
        if A_copy[i][j] == "x":
            A_copy[i][j] = "o"
            # reached[i][j] = 1 はしないでおく(再起の中で絶対1になる)
            landfill(i, j)
        count = sum(map(sum, reached))

        if count == land_count + 1:
            print("YES")
            exit()
print("NO")


from fastapi import HTTPException, status
from typing import List
from sqlalchemy.orm.session import Session
from routers.schemas import Department
from .models import DbDepartment

def get_all(db: Session):
  return db.query(DbDepartment).all()

def create(db: Session, request: Department):
    try:
        new_department = DbDepartment(
            Id=request.id,
            Department=request.department)
        db.add(new_department)
        db.commit()
        db.refresh(new_department)
        return Department
    except Exception as ex:
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(ex))

def create_from_list(db: Session, request: List[Department]):
    list_department = [DbDepartment(Id=department.id,Department=department.department) for department in request]
    try:
        for department in list_department:
            db.add(department)
            db.commit()
            db.refresh(department)
        return request
    except Exception as ex:
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(ex))


from django.shortcuts import redirect, render
from .forms import Registeration

def registeration(request):
    if request.method == 'POST':
        form = Registeration(request.POST)
        if form.is_valid():
            form.save()
            print('New user has been created')
            return redirect('login')
    else:
        form = Registeration()
    context = {
        'form':form
        }
    return render(request, 'users/registeration.html', context)


import cv2
import numpy as np
src = cv2.imread('./data/tomato.jpg',cv2.IMREAD_COLOR)
hsv = cv2.cvtColor(src,cv2.COLOR_BGR2HSV)

h, s, v = cv2.split(hsv)
cv2.imshow('h',h) #색상(Hue) 0~360 -> opencv에서 0~180
cv2.imshow('s',s) #채도(Saturation) 0~255 이미지 색상 깊이 -> opencv에서 0~255
cv2.imshow('v',v) #명도(Value) 0~255 색의 밝고 어두운 정도 -> opencv에서 0~255
cv2.waitKey()
cv2.destroyAllWindows()


"""
Data types for numpy logs and pandas conversion functions.

Author: Patrik Goldschmidt (igoldschmidt@fit.vut.cz)
Author: Jan Kučera (jan.kucera@cesnet.cz)
Date: 2023-04-27
Project: Windower: Feature Extraction for Real-Time DDoS Detection Using ML
Repository: https://github.com/xGoldy/Windower
"""

import numpy as np

from common import defines

# Numpy datatype for internal window statistics
# Currently taking 112B with alignment (structure padding)
NP_DTYPE_WINDOW_STATS = np.dtype([
    # Window summary
    ('window_id', 'u4'),                    # Window identifier
    ('pkts_total', 'u8'),                   # Total number of packets
    ('bytes_total', 'u8'),                  # Total number of bytes
    # Time
    ('tstamp_start', 'u8'),                 # First packet timestamp in the given window
    ('tstamp_end', 'u8'),                   # Last packet timestamp in the given window
    ('pkt_arrivals_avg', 'f8'),             # Average time between packet arrivals
    ('pkt_arrivals_std', 'f8'),             # Std of time between packet arrivals
    # Packet sizes
    ('pkt_size_min', 'u4'),                 # Minimum packet size
    ('pkt_size_max', 'u4'),                 # Maximum packet size
    ('pkt_size_avg', 'f4'),                 # Average of packet sizes
    ('pkt_size_std', 'f4'),                 # Std of packet sizes
    # L4 Protocols
    ('tcp_pkt_count', 'u8'),                # Number of logged TCP packets (segments)
    ('udp_pkt_count', 'u8'),                # Number of logged UDP packets
    ('icmp_pkt_count', 'u8'),               # Number of logged ICMP packets
    # Ports
    ('port_src_unique', 'u4'),              # Number of unique source ports
    ('port_src_entropy', 'f4'),             # Source port entropy
    # Connections
    ('conn_pkts_avg', 'f4'),                # Average number of packets for socket2socket transfers
    # Properties
    ('pkts_frag_count', 'u8'),              # Number of fragmented packets
    ('hdrs_payload_ratio_avg', 'f4'),       # Average of header to whole packet size ratio
], align=True)

NP_DTYPE_WINDOW_AUXDATA = np.dtype([
    ('last_pkt_arrival', 'u8'),             # Timestamp of the last packet arrival
    ('pkt_arrivals_std_aux', 'f4'),         # Aux value for running arrivals std computation
    ('pkt_size_std_aux', 'f4'),             # Aux value for running packet size std computation
], align=True)

# Numpy datatype for windows summary statistics
# Unless explicitly mentioned otherwise, all fields are the average of values from multiple processed windows
NP_DTYPE_WINDOW_SUMMARY_STATS = np.dtype([
    # Window identifier
    (defines.DATA_SRC_IP_COLNAME, 'U46'),   # Source IP address as 46-character length string for IPv6 maximum
    # Window summaries
    ('window_count', 'u4'),                 # Number of summarized windows
    ('window_span', 'u4'),                  # Difference between the last and the first processed window
    ('pkts_total', 'u8'),                   # Total number of packets
    ('bytes_total', 'u8'),                  # Total number of bytes
    ('pkt_rate', 'f4'),                     # Rate of packets per second estimate
    ('byte_rate', 'f4'),                    # Rate of bytes per second estimate
    # Time
    ('pkt_arrivals_avg', 'f8'),             # Average time between packet arrivals
    ('pkt_arrivals_std', 'f8'),             # Std of time between packet arrivals
    # Packet sizes
    ('pkt_size_min', 'u4'),                 # Minimum of all windows packet sizes
    ('pkt_size_max', 'u4'),                 # Maximum of all windows packet sizes
    ('pkt_size_avg', 'f4'),                 # Average of packet sizes
    ('pkt_size_std', 'f4'),                 # Std of packet sizes
    # L4 Protocols
    ('proto_tcp_share', 'f4'),              # TCP traffic share
    ('proto_udp_share', 'f4'),              # UDP traffic share
    ('proto_icmp_share', 'f4'),             # ICMP traffic share
    # Ports
    ('port_src_unique', 'f4'),              # Number of unique source ports
    ('port_src_entropy', 'f4'),             # Source port entropy
    # Connections
    ('conn_pkts_avg', 'f4'),                # Average number of packets for socket2socket transfers
    # Properties
    ('pkts_frag_share', 'f4'),              # Share of the fragmented packets
    ('hdrs_payload_ratio_avg', 'f4'),       # Average of header to whole packet size ratio
], align=True)

# Numpy datatype for inter-window statistics
NP_DTYPE_INTERWINDOW_STATS = np.dtype([
    ('pkts_total_std', 'f4'),               # Std of total number of packets
    ('bytes_total_std', 'f4'),              # Std of total number of bytes
    ('pkt_size_avg_std', 'f4'),             # Std of packet size averages
    ('pkt_size_std_std', 'f4'),             # Std of packet size stds
    ('pkt_arrivals_avg_std', 'f4'),         # Std of packet average time between packet arrivals
    ('port_src_unique_std', 'f4'),          # Std of number of unique source port number
    ('port_src_entropy_std', 'f4'),         # Std of source port entropies
    ('conn_pkts_avg_std', 'f4'),            # Std of number of packets per connection averages
    ('pkts_frag_share_std', 'f4'),          # Std of the fragmented packets share
    ('hdrs_payload_ratio_avg_std', 'f4'),   # Std of header to whole packet ratios
    ('dominant_proto_ratio_std', 'f4'),     # Std of ratio of the dominant L4 protocol
    ('intrawindow_activity_ratio', 'f4'),   # Host activity estimate within the summarized windows
    ('interwindow_activity_ratio', 'f4'),   # Host activity estimate during the whole summarized period
], align=True)

# Approximate size of the numpy array of NP_DTYPE_WINDOW_STATS dtype with a single row
# Initialized upon module load in __init__.py
NP_WINDOW_STATS_ARRAY_SIZE = None


'''
Description: Assignment 3 Question 8 -> Given a collection of at most 10 symbols defining an ordered alphabet,
                                        and a positive integer n (n≤10).
                                        Return all strings of length n that can be formed from the alphabet,
                                        ordered lexicographically
Author: Masood Ahmed
UID: 3035812127

'''
# importing itertools to get repeated permutations
import itertools

'''
Description: this function finds permutations of the strings with k characters
parameters: listOfChar --> a list of all the characters
           k --> the length of each permutation

return:  a list of all the possible permutations
'''
def finding_permutations(listOfChar,k):
    final = []
    for p in itertools.product(listOfChar, repeat=k):
        p = ''.join(p)
        final.append(p)

    final.sort()
    return final

# main program area
if __name__ == "__main__":
    # Assuming that input is in the following form:
    # A C G T
    # 2

    inputFile = open('input.txt', 'r')
    lines = []
    for line in inputFile.readlines():
        line = line.rstrip('\n')
        lines.append(line)

    listOfChar = lines[0].split(" ")
    k = int(lines[-1])
    inputFile.close()
    result = finding_permutations(listOfChar,k)
    outputfile = open('outputQ8.txt', 'w')
    for each in result:
        outputfile.write(each + '\n')
    outputfile.close()


"""
Module instead for bot command "/weather"
"""

from telebot.types import Message, CallbackQuery
from loader import bot
from database.functions import get_user_city
from .city import change_city
from utils import weather_requests
from keyboards.inline import inline_keyboad_by_dict
from datetime import datetime as dt

@bot.message_handler(commands=["weather"])
def weather_start(message: Message) -> None:
    """
    Message handler which catching the command "/weather".
    Send a keyboard with choice of view a weather

    Args:
        message (Message): user's message
    """
    city_info = get_user_city(message.chat.id)
    if city_info:
        keyboard = inline_keyboad_by_dict(
            {
                "Погода сьогодні": "today",
                "Наступні 24 години (погодинно)": "hourly",
                "10 днів": "10 days",
            }
        )
        bot.send_message(
            message.chat.id, "Як саме хочете переглядати погоду?", reply_markup=keyboard
        )
    else:
        change_city(message)

@bot.callback_query_handler(func=lambda data: data.data in ("today", "hourly", "10 days"))
def get_weather(callback: CallbackQuery):
    """
    Function which catching callback from keyboard of choice the view a weather and returned a message with weather
    which user selected earlier.

    Args:
        callback (CallbackQuery): callback from inline keyboard
    """
    msg = bot.send_message(callback.message.chat.id, "Одну секунду, дізнаюсь погоду...")
    city = get_user_city(callback.from_user.id)

    if callback.data == "hourly":
        weather = weather_requests.get_weather_hourly(
            city.get("latitude"), city.get("longitude")
        )
        bot.delete_message(msg.chat.id, msg.id)
        if weather:
            text = ""
            text_template = (
                "{smile} {time} ({day})\n"
                "🌡 Температура: {temp}°С, відчувається як {temp_like}°С\n"
                "💦 Вологість повітря: {humidity}%\n"
                "🔆 УФ-індекс: {uv_index} з 10 ({uv_description})\n"
                "Опади: {precip_type} {precip_chance}% {precip_quantity}мм\n\n"
            )

            for hour in range(24):
                precip = weather.get("precipType")[hour]

                text += text_template.format(
                    smile="☀" if weather.get("dayOrNight")[hour] == "D" else "🌙",
                    time=dt.fromisoformat(weather.get("validTimeLocal")[hour][:-5]).strftime("%H:%M"),
                    day=weather.get("dayOfWeek")[hour],
                    temp=weather.get("temperature")[hour],
                    temp_like=weather.get("temperatureFeelsLike")[hour],
                    humidity=weather.get("relativeHumidity")[hour],
                    uv_index=weather.get("uvIndex")[hour],
                    uv_description=weather.get("uvDescription")[hour],
                    precip_type="Дощ 💧" if precip == "rain" else "Сніг ❄",
                    precip_chance=weather.get("precipChance")[hour],
                    precip_quantity=weather.get("qpf")[hour] if precip == "rain" else weather.get("qpfSnow")[hour]
                )
        else:
            text = "Сталася непередбачувана помилка 😳"
        bot.send_message(callback.message.chat.id, text)

    elif callback.data == "today":
        weather = weather_requests.get_today_weather(
            city.get("latitude"), city.get("longitude")
        )
        bot.delete_message(msg.chat.id, msg.id)
        if weather:
            text = (
                "{day_part} Погода сьогодні ({day}) в {city_name}\n\n"
                "{description}\n\n"
                "⬇ Тиск: {pressure} мбар\n\n"
                "🌡 Температура зараз: {temp}°С, відчувається як {temp_like}°С\n\n"
                "🌡 Мінімальна: {temp_min}°С, максимальна {temp_max}°С\n\n"
                "💦 Вологість повітря: {humidity}%\n\n"
                "🔆 УФ-індекс: {uv_index} з 10\n\n"
                "👀 Видимість: {visibility}\n\n"
                "💧 Точка роси: {dew_point}°С"
            ).format(
                day_part="☀" if weather.get("dayOrNight") == "D" else "🌙",
                day=weather.get("dayOfWeek"),
                city_name=city.get("city_name"),
                description=weather.get("wxPhraseLong"),
                pressure=weather.get("pressureMeanSeaLevel"),
                temp=weather.get("temperature"),
                temp_like=weather.get("temperatureFeelsLike"),
                temp_min=weather.get("temperatureMin24Hour"),
                temp_max=weather.get("temperatureMax24Hour"),
                humidity=weather.get("relativeHumidity"),
                uv_index=weather.get("uvIndex"),
                visibility=weather.get("visibility"),
                dew_point=weather.get("temperatureDewPoint")
            )
        else:
            text = "Сталася непередбачувана помилка 😳"

        bot.send_message(callback.message.chat.id, text)

    elif callback.data == "10 days":
        weather = weather_requests.get_10_days_weather(
            city.get("latitude"), city.get("longitude")
        )
        bot.delete_message(msg.chat.id, msg.id)
        if weather:
            text_template = (
                "{smile} {daypart_name}\n"
                "🌡 Температура: {temp}°С\n"
                "☁ Хмарність: {clouds}%\n"
                "🔆 УФ-індекс: {uv_index} з 10\n"
                "Опади: {precip_type} {precip_chance}% {precip_quantity}мм\n"
                "{description}\n\n"
            )

            for day in range(0, 20, 2):
                text = ""
                night = day + 1

                if weather.get("dayOrNight")[day]:
                    precip = weather.get("precipType")[day]
                    text += text_template.format(
                        daypart_name=weather.get("daypartName")[day],
                        smile="☀",
                        temp=weather.get("temperature")[day],
                        clouds=weather.get("cloudCover")[day],
                        uv_index=weather.get("uvIndex")[day],
                        precip_chance=weather.get("precipChance")[day],
                        precip_type="Дощ 💧" if precip == "rain" else "Сніг ❄",
                        precip_quantity=weather.get("qpf")[day] if precip == 'rain'
                        else weather.get("qpfSnow")[day],
                        description=weather.get("wxPhraseLong")[day]
                    )

                if weather.get("dayOrNight")[night]:
                    precip = weather.get("precipType")[night]
                    text += text_template.format(
                        daypart_name=weather.get("daypartName")[night],
                        smile="🌙",
                        temp=weather.get("temperature")[night],
                        clouds=weather.get("cloudCover")[night],
                        uv_index=weather.get("uvIndex")[night],
                        precip_chance=weather.get("precipChance")[night],
                        precip_type="Дощ 💧" if weather.get("precipType")[night] == "rain" else "Сніг ❄",
                        precip_quantity=weather.get("qpf")[night] if precip == 'rain'
                        else weather.get("qpfSnow")[night],
                        description=weather.get("wxPhraseLong")[night],
                    )
                bot.send_message(callback.message.chat.id, text)
        else:
            bot.send_message(callback.message.chat.id, "Сталася непередбачувана помилка 😳")
    weather_start(callback.message)


#!/usr/bin/env python3

#### PYTHON IMPORTS ############################################################
import argparse
import csv
import json
import numpy as NP
import os
import sys
import warnings

from collections import Counter

#### SCIKITLEARN IMPORTS #######################################################
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
warnings.filterwarnings("ignore", category=UserWarning)

#### GLOBALS ###################################################################
CURR_PATH = os.path.dirname(os.path.realpath(__file__))
DATA_PATH = os.path.join(CURR_PATH, "data")
MODELS_PATH = os.path.join(CURR_PATH, "models")
RESULTS_PATH = os.path.join(CURR_PATH, "results")

#### HELPER FUNCTIONS ##########################################################
def readData(filename):
    """
    Read data from a CSV file. The CSV file should have the following 4 columns:
      1 ID          Unique ID for event
      2 HOST        The hostname that the Splunk event ocurred on
      3 HOST_TYPE   One of "Attacker" or "Victim"
      4 EVENT       Raw text of a Splunk event (log)
      5 TACTIC      One or more MITRE ATT&CK Tactics
      6 TECHNIQUE   One or more MITRE ATT&CK Techniques

    GIVEN:
      filename (str)    file path to read data from

    RETURN:
      data (list)       list of events from CSV file
      hosts (list)      list of host types for each event
      labels (dict)     dictionary with two keys, "tactics" and "techniques";
                        each value is a list of human-annotated labels
    """
    data = list()
    hosts = list()
    labels = {"tactics": list(), "techniques": list()}
    with open(filename, newline="") as csvfile:
        csv_reader = csv.reader(csvfile, delimiter=",", quotechar="\"")

        # Skip header row
        next(csv_reader)

        # Read data from CSV
        for row in csv_reader:
            data.append(row[3])
            hosts.append(row[2])
            labels["tactics"].append(row[4])
            labels["techniques"].append(row[5])

    print("Read {} data points from '{}'.".format(len(data), filename))
    return data, hosts, labels

def printStats(data, labels):
    """ Print useful information about the data. """
    cnt_tactics = Counter(labels["tactics"])
    k_tactics = list(cnt_tactics.keys())
    v_tactics = list(cnt_tactics.values())
    cnt_techniques = Counter(labels["techniques"])
    k_techniques = list(cnt_techniques.keys())
    v_techniques = list(cnt_techniques.values())

    print("DATASET INFO:")
    print("  Data Points: {}".format(len(data)))
    print("  TACTICS:")
    for i in range(0, len(k_tactics)):
        print("    '{}': {}".format(k_tactics[i], v_tactics[i]))
    print("  TECHNIQUES:")
    for i in range(0, len(k_techniques)):
        print("    '{}': {}".format(k_techniques[i], v_techniques[i]))

def preprocessData(data):
    """
    Clean up the data by replacing timestamps and IP addresses with unique
    tokens.

    GIVEN:
      data (list)       list of events

    RETURN:
      new_data (list)   copy of `data` with timestamps and IP addresses
                        cleaned up
    """
    new_data = data

    # Replace timestamps with '<<TIMESTAMP>>'
    # Replace IP addresses with '<<ATTACKER>>' or '<<DEFENDER>>'

    return new_data

def augmentData(data, hosts, labels, target_class, append_states, append_hosts):
    """
    Add the Tactic or Technique to the data. Add host type to the data.

    GIVEN:
      data (list)           list of events
      hosts (list)          one of ["Attacker", "Victim"]
      labels (dict)         human-annotated labels for data
      target_class (str)    one of ["tactics", "techniques"]
      append_states (bool)  whether or not to append tactics/techniques to the data
      append_hosts (bool)   whether or not to append hosts types to the data

    RETURN:
      new_data (list)       copy of `data` with tactics/techniques and/or host types appended
    """
    new_data = data
    target_swap = {"tactics": "techniques", "techniques": "tactics"}

    if append_hosts:
        for i in range(0, len(data)):
            new_data[i] = new_data[i] + " " + hosts[i]

    if append_states:
        for i in range(0, len(data)):
            new_data[i] = new_data[i] + " " + labels[target_swap[target_class]][i]

    return new_data

def removeSingles(data, hosts, labels, target_class):
    """
    Remove entries from the dataset where the tactic/technique only occurs once.

    GIVEN:
      data (list)           list of events
      hosts (list)          one of ["Attacker", "Victim"]
      labels (dict)         human-annotated labels for data
      target_class (str)    one of ["tactics", "techniques"]

    RETURN:
      new_data (list)       copy of data with single occurrences removed
      new_hosts (list)      copy of hosts with single occurrences removed
      new_labels (dict)     copy of labels with single occurrences removed
    """
    new_data = list()
    new_hosts = list()
    new_labels = {"tactics": list(), "techniques": list()}

    counts = dict(Counter(labels[target_class]))
    singles = list()
    for k, v in counts.items():
        if v == 1:
            singles.append(k)

    for i in range(0, len(data)):
        if labels[target_class][i] not in singles:
            new_data.append(data[i])
            new_hosts.append(hosts[i])
            new_labels["tactics"].append(labels["tactics"][i])
            new_labels["techniques"].append(labels["techniques"][i])
        else:
            pass

    return new_data, new_hosts, new_labels

def trainTestSplit(data, labels, target_class, test_size):
    """
    Split the data and it's labels into training and testing sets.

    GIVEN:
      data (list)           list of events
      labels (dict)         human-annotated labels for data
      target_class (str)    one of ["tactics", "techniques"]
      test_size (double)    percentage to hold out for testing

    RETURN:
      X_train (list)    list of training data values
      X_test (list)     list of testing data values
      Y_train (list)    list of training data labels
      Y_test (list)     list of testing data labels
    """
    X_train, X_test, Y_train, Y_test = train_test_split(
        data, labels[target_class], test_size=test_size,
        #stratify=labels[target_class]
    )

    return X_train, X_test, Y_train, Y_test

#### TEST FUNCTIONS ############################################################
def testMNB(data, labels, target_class, test_size, trial_prefix):
    """
    Train and test the performance of MultinomialNB with various parameter
    combinations.
    """
    X_train, X_test, Y_train, Y_test = trainTestSplit(
        data, labels, target_class, test_size)

    pipe_MNB = Pipeline([
        ("vect", CountVectorizer()),
        ("tfidf", TfidfTransformer()),
        ("clf", MultinomialNB())
    ])

    # All combinations of these parameters will be tested
    GS_MNB_params = {
        "vect__ngram_range": [(1, 1), (1, 2), (1, 3), (2, 2), (2, 3), (3, 3)],
        "vect__stop_words": (None, "english"),
        "vect__strip_accents": (None, "ascii", "unicode"),
        "vect__analyzer": ("word", "char", "char_wb"),
        "vect__lowercase": (True, False),
        "tfidf__use_idf": (True, False),
        "clf__alpha": (0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99),
        "clf__fit_prior": (True, False)
    }

    # Test all combinations of GS_MNB_params using 10-fold CV and all available CPU's
    pipe_GS_MNB = GridSearchCV(pipe_MNB, GS_MNB_params, cv=10, n_jobs=-1, verbose=1)
    pipe_GS_MNB.fit(X_train, Y_train)

    best_params = pipe_GS_MNB.best_params_
    print("Best Score for MultinomialNB:")
    print("    {}".format(pipe_GS_MNB.best_score_))
    print("Best Parameters for MultinomialNB:")
    for p in sorted(GS_MNB_params.keys()):
        print("    {}: {}".format(p, best_params[p]))

    pipe_GS_MNB_predicted = pipe_GS_MNB.predict(X_test)
    print(classification_report(Y_test, pipe_GS_MNB_predicted))

    # Dump best model to disk
    outfile = os.path.join(MODELS_PATH, "{}_MultinomialNB_{}_best.pkl".format(trial_prefix, target_class))
    joblib.dump(pipe_GS_MNB.best_estimator_, outfile)
    # Dump best model parameters to disk
    params_outfile = os.path.join(MODELS_PATH, "{}_MultinomialNB_{}_best_params.json".format(trial_prefix, target_class))
    f = open(params_outfile, "w")
    f.write(json.dumps(best_params))
    f.close()

def testLSVC(data, labels, target_class, test_size, trial_prefix):
    """
    Train and test the performance of LinearSVC with various parameter
    combinations.
    """
    X_train, X_test, Y_train, Y_test = trainTestSplit(
        data, labels, target_class, test_size)

    pipe_LSVC = Pipeline([
        ("vect", CountVectorizer()),
        ("tfidf", TfidfTransformer()),
        ("clf", LinearSVC())
    ])

    # All combinations of these parameters will be tested
    GS_LSVC_params = {
        "vect__ngram_range": [(1, 1), (1, 2), (1, 3), (2, 2), (2, 3), (3, 3)],
        "vect__stop_words": (None, "english"),
        "vect__strip_accents": (None, "ascii", "unicode"),
        "vect__analyzer": ("word", "char", "char_wb"),
        "vect__lowercase": (True, False),
        "tfidf__use_idf": (True, False),
        "clf__penalty": ("l1", "l2"),
        "clf__loss": ("hinge", "squared_hinge"),
        "clf__dual": (True, False),
        "clf__tol": (1e-1, 1e-4, 1e-10),
        "clf__multi_class": ("ovr", "crammer_singer"),
        "clf__fit_intercept": (True, False),
        "clf__max_iter": (1000, 5000)
    }

    # Test all combinations of GS_LSV_params using 10-fold CV and all available CPU's
    pipe_GS_LSVC = GridSearchCV(pipe_LSVC, GS_LSVC_params, cv=10, n_jobs=-1, verbose=1)
    pipe_GS_LSVC.fit(X_train, Y_train)

    best_params = pipe_GS_LSVC.best_params_
    print("Best Score for LinearSVC:")
    print("    {}".format(pipe_GS_LSVC.best_score_))
    print("Best Parameters for LinearSVC:")
    for p in sorted(GS_LSVC_params.keys()):
        print("    {}: {}".format(p, best_params[p]))

    pipe_GS_LSVC_predicted = pipe_GS_LSVC.predict(X_test)
    print(classification_report(Y_test, pipe_GS_LSVC_predicted))

    # Dump best model to disk
    outfile = os.path.join(MODELS_PATH, "{}_LinearSVC_{}_best.pkl".format(trial_prefix, target_class))
    joblib.dump(pipe_GS_LSVC.best_estimator_, outfile)
    # Dump best model parameters to disk
    params_outfile = os.path.join(MODELS_PATH, "{}_LinearSVC_{}_best_params.json".format(trial_prefix, target_class))
    f = open(params_outfile, "w")
    f.write(json.dumps(best_params))
    f.close()

#### TRAIN FUNCTIONS ##########################################################
def trainBest(data, labels, target_class, model_type, trial_prefix):
    """
    Re-train the best model using all available training data.

    GIVEN:
      data (list)           list of events
      labels (dict)         human-annotated labels for data
      target_class (str)    one of ["tactics", "techniques"]
      model_type (str)      one of ["nb", "lsvc"] denoting which type of model
                            to build
      trial_prefix (int)    best trial to use for re-training
    """

    # Determine best model parameters to use
    model_map = {"nb": "MultinomalNB", "lsvc": "LinearSVC"}
    best_params_file = "{}/{}_{}_{}_best_params.json".format(
        trial_prefix, trial_prefix, model_map[model_type], target_class)
    # Load best model params from disk
    best_params = dict()
    with open(os.path.join(MODELS_PATH, best_params_file), "r") as f:
        best_params = json.load(f)

    # Train new model with best params
    pipe_best = Pipeline([
        ("vect", CountVectorizer(
            analyzer=best_params["vect__analyzer"],
            lowercase=best_params["vect__lowercase"],
            ngram_range=best_params["vect__ngram_range"],
            stop_words=best_params["vect__stop_words"],
            strip_accents=best_params["vect__strip_accents"])),
        ("tfidf", TfidfTransformer(
            use_idf=best_params["tfidf__use_idf"])),
        ("clf", LinearSVC(
            dual=best_params["clf__dual"],
            fit_intercept=best_params["clf__fit_intercept"],
            loss=best_params["clf__loss"],
            max_iter=best_params["clf__max_iter"],
            multi_class=best_params["clf__multi_class"],
            penalty=best_params["clf__penalty"],
            tol=best_params["clf__tol"]))
    ])
    pipe_best.fit(data, labels[target_class])

    # Dump best model to disk
    outfile = os.path.join(MODELS_PATH, "best_{}_{}.pkl".format(model_map[model_type], target_class))
    joblib.dump(pipe_best, outfile)
    # Dump best model parameters to disk
    params_outfile = os.path.join(MODELS_PATH, "best_{}_{}_params.json".format(model_map[model_type], target_class))
    f = open(params_outfile, "w")
    f.write(json.dumps(best_params))
    f.close()

    return pipe_best, best_params

#### CLASSIFY FUNCTIONS ########################################################
def classify(data, target_class, model_type, best_model, data_path):
    """
    Classify the data using the best model.

    GIVEN:
      data (ist)            list of events to be classified
      target_class (str)    one of ["tactics", "techniques"]
      model_type (str)      one of ["nb", "lsvc"]
      best_model (model)    the best model
      data_path (str)       relative path for data being classified
    """

    # Classify new data using best model
    model_map = {"nb": "MultinomalNB", "lsvc": "LinearSVC"}
    print("Classifying {} for new data with best {} model.".format(
        target_class, model_map[model_type]))
    predictions = best_model.predict(data)

    # Write predictions to disk
    data_name = data_path.split("/")[-1].rstrip(".csv")
    print(data_name)
    outfile = "predictions_{}_{}_{}.csv".format(
        model_map[model_type], target_class, data_name)
    with open(outfile, "w") as f:
        for pred in predictions:
            f.write(pred + "\n")

#### MAIN ######################################################################
if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Map Splunk logs to MITRE ATT&CK states with Scikit-Learn."
    )
    parser.add_argument(
        "--append_states", type=bool, choices=[True, False], default=False,
        help="Whether or not to append tactics/techniques to the raw data "
        "before testing/training. Default: False."
    )
    parser.add_argument(
        "--append_hosts", type=bool, choices=[True, False], default=False,
        help="Whether or not to append host types to the raw data before "
        "testing/training. Default: False."
    )
    parser.add_argument(
        "--model_type", choices=["nb", "lsvc"],
        help="The type of model to use. Either Multinomial Naive Bayes (nb) "
        "or Linear Support Vector Classifier (lsvc)."
    )
    parser.add_argument(
        "--target", choices=["tactics", "techniques"],
        help="The target class for testing, training, and classification."
    )
    parser.add_argument(
        "--trial_prefix", type=str, default="",
        help="String to prefix to model files saved to disk."
    )
    parser.add_argument(
        "--ignore_singles", type=bool, choices=[True, False], default=False,
        help="Whether or not to remove tactics/techniques that only occur once "
        "from the dataset."
    )
    parser.add_argument(
        "--classify_dataset", type=str,
        help="Relative path to dataset (CSV) to use for classification."
    )
    parser.add_argument(
        "command", choices=["info", "test", "train", "classify"],
        help="The command to run. Info prints stats for the specified dataset."
        " Test does 10-fold CV to find the best feature combinations. Train "
        "builds a new model using all data and the best feature set. Classify "
        "uses the best model to classify a new dataset."
    )
    parser.add_argument(
        "dataset", type=str, help="Relative path to the dataset (CSV) to use."
    )
    args = parser.parse_args()
    print(args)

    if args.command == "info":
        if args.target is None:
            sys.exit("Argument '--target' required for 'command=info'.")

        data, hosts, labels = readData(args.dataset)
        if args.ignore_singles == True:
            data, hosts, labels = removeSingles(data, hosts, labels, args.target)
        printStats(data, labels)
    elif args.command == "test":
        if args.model_type is None or args.target is None:
            sys.exit("Arguments '--model_type' and '--target' required for "
                     "'command=test'.")

        data, hosts, labels = readData(args.dataset)
        if args.ignore_singles == True:
            data, hosts, labels = removeSingles(data, hosts, labels, args.target)
        data = augmentData(data, hosts, labels, args.target, args.append_states, args.append_hosts)

        if args.model_type == "nb":
            testMNB(data, labels, args.target, 0.33, args.trial_prefix)
        elif args.model_type == "lsvc":
            testLSVC(data, labels, args.target, 0.33, args.trial_prefix)
    elif args.command == "train":
        if args.target is None or args.model_type is None or args.trial_prefix is None:
            sys.exit("Arguments '--model_type', '--target', and '--trial_prefix'"
                " required for 'command=train'.")

        data, hosts, labels = readData(args.dataset)
        if args.ignore_singles == True:
            data, hosts, labels = removeSingles(data, hosts, labels, args.target)
        data = augmentData(data, hosts, labels, args.target, args.append_states, args.append_hosts)
        trainBest(data, labels, args.target, args.model_type, args.trial_prefix)
    elif args.command == "classify":
        if args.target is None or args.model_type is None or args.trial_prefix is None or args.classify_dataset is None:
            sys.exit("Arguments '--model_type', '--target', '--trial_prefix', "
                    "and '--classify_dataset' required for 'command=classify'.")

        data, hosts, labels = readData(args.dataset)
        if args.ignore_singles == True:
            data, hosts, labels = removeSingles(data, hosts, labels, args.target)
        data = augmentData(data, hosts, labels, args.target, args.append_states, args.append_hosts)
        best_model, best_params = trainBest(data, labels, args.target, args.model_type, args.trial_prefix)

        data, hosts, labels = readData(args.classify_dataset)
        if args.ignore_singles == True:
            data, hosts, labels = removeSingles(data, hosts, labels, args.target)
        data = augmentData(data, hosts, labels, args.target, args.append_states, args.append_hosts)
        classify(data, args.target, args.model_type, best_model, args.classify_dataset)


import numpy as np
from datetime import datetime
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

def loadGeneralTrainData():
    # Load general training data
    path = "./data/generalTrainingData/"
    subjectFilename = ["subject1.csv", "subject4.csv", "subject6.csv", "subject9.csv"]
    indexFilename = ["index1.csv", "index4.csv", "index6.csv", "index9.csv"]

    data = []
    indexes = []

    for filename in subjectFilename:
        data.append(loadData(path + filename, True))

    for filename in indexFilename:
        indexes.append(loadData(path + filename))

    return data, indexes

def loadGeneralTestData():
    # Load general testing data set
    path = "./data/testData/finalTest/general_test_instances.csv"
    return loadData(path)[:, 7:]

def loadSingleTrainData():
    # Load single traning data
    path = "./data/singleTrainingData/"
    data2 = loadData(path + "subject2_training.csv", True)
    data7 = loadData(path + "subject7_training.csv", True)
    index2 = loadData(path + "index2_training.csv")
    index7 = loadData(path + "index7_training.csv")

    return data2, data7, index2, index7

def loadSingleTestData():
    # Load single testing data set
    path = ["./data/testData/finalTest/subject2_instances.csv",
        "./data/testData/finalTest/subject7_instances.csv"]
    return loadData(path[0])[:, 7:], loadData(path[1])[:, 7:]

def generalTransform(data, indexes):
    # Transform data for general training set
    Xs, Ys = [], []

    for subjectIdx in range(4):
        indexList = indexes[subjectIdx]
        subjectData = data[subjectIdx]
        X, Y = transform(subjectData, indexList)

        Xs.append(X)
        Ys.append(Y)

    return Xs, Ys

def singleTransform(data2, data7, index2, index7):
    # Transform data for single training set
    X2, Y2 = transform(data2, index2)
    X7, Y7 = transform(data7, index7)
    return X2, X7, Y2, Y7

def transform(subjectData, indexList):
    # utility function to transform data into 63 feature instances
    X, Y = [], []

    for idx in range(0, len(indexList) - 6):
        if (indexList[idx] == indexList[idx + 6] - 6):

            instance = subjectData[idx: idx + 7].flatten('F')
            X.append(instance[7:63])
            Y.append(instance[-1])

    return np.array(X), np.array(Y)

def splitTrainTest(Xs, Ys, idx):
    # lambda function for concatenate numpy arrays
    f = lambda a, b: np.concatenate((a, b), axis=0)

    # split the training and testing data set
    X = Xs[ :idx] + Xs[idx+1:]
    Y = Ys[:idx] + Ys[idx+1:]
    X = reduce(f, X)
    Y = reduce(f, Y)
    Xt = Xs[idx]
    Yt = Ys[idx]

    return X, Y, Xt, Yt

def sampling(X, Y):
    # Sample more positive data
    posSamples = []

    for idx, val in enumerate(Y):
        if val == 1:
            posSamples.append(X[idx])

    posSamples = np.array(posSamples)

    numSamples = X.shape[0] - posSamples.shape[0]
    mean =  np.mean(posSamples, axis=0)
    std = np.std(posSamples, axis=0)

    randomSamples = np.random.normal(mean, std, (numSamples, X.shape[1]))
    X = np.concatenate((X, randomSamples), axis=0)

    ones = np.ones(numSamples).reshape(numSamples)
    Y = np.concatenate((Y, ones), axis=0)

    return X, Y

def pca(X, Xt):
    # function for principle component analysis
    pca = PCA(.95)
    pca.fit(X)
    pcaX = pca.transform(X)
    pcaXt = pca.transform(Xt)

    return pcaX, pcaXt

def scaler(X, Xt):
    # function for preprocessing
    scaler = StandardScaler()
    scaler.fit(X)
    stdX = scaler.transform(X)
    stdXt = scaler.transform(Xt)

    return stdX, stdXt

def loadData(path, convertDate=False):
    # load data from file and return data matrix
    file = open(path, "r")

    if convertDate:
        dateStr2Hour = lambda x: float(datetime.strptime(x.decode("utf-8"), '%Y-%m-%dT%H:%M:%SZ').hour)
        data = np.genfromtxt(file, delimiter=",", converters={0: dateStr2Hour})

    else:
        data = np.genfromtxt(file, delimiter=",")

    return data

def printAccuracy(YPredict, Yt, title):
    #print(YPredict)
    # Convert YPredict to binary values, and count the number of 1s
    f = lambda a: 0 if a < 0.5 else 1
    YPredict = map(f, YPredict)
    count0 = YPredict.count(0)
    count1 = YPredict.count(1)

    TP, FP, FN, TN = 0, 0, 0, 0

    for i in range(len(Yt) - 1):
        if YPredict[i] == 1 and Yt[i] == 1:
            TP = TP + 1
        elif YPredict[i] == 1 and Yt[i] == 0:
            FP = FP + 1
        elif YPredict[i] == 0 and Yt[i] == 1:
            FN = FN + 1
        else:
            TN = TN + 1

    print("\n==> {0}".format(title))
    print("# of 0: {0}, # of 1: {1}".format(str(count0), str(count1)))
    print("TP: {0}, FP: {1}, FN: {2}, TN: {3}".format(str(TP), str(FP), str(FN), str(TN)))

def reduceProb(pred, prob):
    # Remove useles value in package generated probability
    newProb = []

    for idx, val in enumerate(pred):
        if val == 0:
            newProb.append(prob[idx][0])
        else:
            newProb.append(prob[idx][1])

    return np.array(newProb)

def savePred(YPredict, YProb, filename):
    # Save prediction files
    f1 = lambda a: 0 if a < 0 else (1 if a > 1 else a)
    f2 = lambda a: 0 if a < 0.5 else 1
    proba = map(f1, YProb)
    YPredict = map(f2, YPredict)
    output = np.vstack((proba, YPredict)).T
    np.savetxt(filename, output, fmt='%0.12f,%i', delimiter=',')

def saveGroundTruth(Y, filename):
    # Save ground truth to csv file
    np.savetxt(filename, Y.astype(int), fmt='%i', delimiter=' ')