'''
Created on 2021-07-31

@author: wf
'''
import unittest
from tests.datasourcetoolbox import DataSourceTest
from corpus.lookup import CorpusLookup
from corpus.event import EventStorage
from lodstorage.query import Query
import copy
import json

class TestStatistics(DataSourceTest):
    '''
    test statistics
    '''

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.lookup=CorpusLookup()

    def setUp(self, debug:bool=False, profile:bool=True, **kwargs):
        DataSourceTest.setUp(self, debug=debug, profile=profile, **kwargs)
        self.lookup=TestStatistics.lookup
        pass

    def testIssue255(self):
        '''
        test issue 255 problem list
        '''
        show=False
        lookup=self.lookup
        sqlQuery="""select url,endDate-startdate+1 as duration from
event_orclone
where endDate is not Null and startDate is not Null
and duration<0 or  duration>7
order by duration"""
        query=Query(name="issue #255",query=sqlQuery,lang='sql')
        qlod=lookup.getLod4Query(query.query)
        prefix="https://confident.dbis.rwth-aachen.de/or/index.php?title="
        for tablefmt in ["mediawiki","github","latex"]:
            lod=copy.deepcopy(qlod)
            query.prefixToLink(lod, prefix, tablefmt)
            qdoc=query.documentQueryResult(lod,tablefmt=tablefmt)
            if show:
                print(qdoc)

    def testMultiQuery(self):
        '''
        test MultiQuery
        '''
        lookup=self.lookup
        multiQuery="""SELECT count(*) AS count,ordinal
        FROM {event}
        GROUP BY ordinal ORDER by 1"""
        dictOfLod=lookup.getDictOfLod4MultiQuery(multiQuery)
        debug=self.debug
        #debug=True
        if debug:
            jsonStr=json.dumps(dictOfLod, sort_keys=True, indent=2,default=str)
            print(jsonStr)

    def testStatistics(self):
        '''
        test statistics
        '''
        lookup=self.lookup
        qm=EventStorage.getQueryManager()
        self.assertIsNotNone(qm)
        self.assertTrue(len(qm.queriesByName)>1)
        showMarkup=False
        # showMarkup=True
        failCount=0
        for _name,query in qm.queriesByName.items():
            try:
                qlod=lookup.getLod4Query(query.query)
                for tablefmt in ["mediawiki","github","latex"]:
                    qdoc=query.documentQueryResult(qlod,tablefmt=tablefmt)
                    if showMarkup:
                        print(qdoc)
            except Exception as ex:
                print(f"query: {query.query} failed:\n{ex}")
                failCount+=1
        self.assertTrue(failCount<=3)
        pass

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()


from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
from pathlib import Path
import pandas as pd
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from itertools import chain
import os
import time

def delete_old_file(file):
    if os.path.exists(file):
        os.remove(file)
    else:
        print("The file does not exist")

def configure_driver():
    # Add additional Options to the webdriver
    chrome_options = Options()
    # add the argument and make the browser Headless.
    chrome_options.add_argument("--headless")
    #finds and use local chromedriver
    d = Path(__file__).resolve().parents[0]
    driver = webdriver.Chrome(executable_path=str(d)+ "/chromedriver", options = chrome_options)
    return driver

def getData(driver,file,arrow_count):
    #Get Page
    driver.get(f"https://finance.yahoo.com/quote/%5EGSPC/history?period1=-1325635200&period2=1603929600&interval=1mo&filter=history&frequency=1mo&includeAdjustedClose=true")
    try:
        #Simulate Key Down to Load Entire Page
        actions = ActionChains(driver)
        actions.send_keys(Keys.ARROW_DOWN * arrow_count)
        actions.perform()
    except TimeoutException:
        print("TimeoutException: Element not found")
        return None

    #Create a parse tree of page sources after searching
    soup = BeautifulSoup(driver.page_source, "lxml")

    #Find the table element
    table = soup.find('table')

    #Seperate Headers
    headerList=[]
    table_headers = table.find_all('th')
    for th in table_headers:
        col = th.find_all('span')
        row = [i.text for i in col]
        headerList.append(row)

    headerList=list(chain.from_iterable(headerList))

    #Seperate Rows
    rowList=[]
    table_rows = table.find_all('tr')
    for tr in table_rows:
        td = tr.find_all('td')
        row = [i.text for i in td]
        rowList.append(row)

    #Emit CSV
    df = pd.DataFrame(rowList,columns=headerList)
    df.to_csv (file, index = False, header=True)
    return (df)

#Arrow Count Config (for test)
arrow_count=50000
#arrow_count=1
#FileName Config
file_name="data/SPHistory.csv"
#Get Start Time Timing
start_time = time.time()
#delete old file
delete_old_file(file_name)
# create the driver object.
driver = configure_driver()
#search_keyword = "Web Scraping"
df=getData(driver,file_name,arrow_count)
# close the driver.
driver.close()

#df= pd.read_csv(FileName)
print("Compete!!" +"\n\n" +
      "Rows Scraped:" + "\n" +
      str(df.count()) +  "\n\n"  +
      "Time Elapsed" +"\n" +
      "--- %s seconds ---" % round((time.time() - start_time),2)
        )


# -*- coding: utf-8 -*-
import sys
import multiprocessing
from multiprocessing import *
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures import as_completed
import platform
import time
import cv2
from pprint import pprint

from utils.pretty_print import *

def load_images(count, image_filepaths):
    start_time = time.time()

    # images = [cv2.imread(f_input, cv2.IMREAD_COLOR) for f_input in image_list]
    loaded_images = [None] * count

    if count > len(image_filepaths):
        print_red(f"Error: load_images: {count} > {len(image_filepaths)}")

    cpu_count = multiprocessing.cpu_count()
    cpu_count = min(int(cpu_count * (3/4)), cpu_count-2)

    if True:
        with ThreadPoolExecutor(max_workers=min(cpu_count, count)) as executor:
            works = [[no, image_filepaths[no]] for no in range(count)]
            futures = [executor.submit(__load_image, w) for w in works]
            for future in concurrent.futures.as_completed(futures):
                (i, img) = future.result()
                loaded_images[i] = img

    else:
        worker_count = int(count / cpu_count)
        img_count = cpu_count * worker_count
        remaining_img_count = count - img_count

        with ProcessPoolExecutor(worker_count) as executor:
            futures = list()

            for i in range(0, img_count, worker_count):
                futures.append(executor.submit(__load_images, [[no, image_filepaths[no]] for no in range(i, i+worker_count)]))

            # Put the img in the list
            for future in as_completed(futures):
                i_imgs = future.result()
                for (no, img) in i_imgs:
                    loaded_images[no] = img
                    # print(f"loaded image no. {no}")

        if remaining_img_count > 0:
            i_imgs = __load_images([[no, image_filepaths[no]] for no in range(img_count, count)])
            for (no, img) in i_imgs:
                loaded_images[no] = img

    elapsed_time = time.time() - start_time
    print(f"loaded {count} images in {elapsed_time:.02f}s (cpu count={cpu_count})")
    # sys.exit()

    return loaded_images

def __load_images(i_fps):
    # List of [i, image_filepath]
    with ThreadPoolExecutor(len(i_fps)) as executor:
        futures = [executor.submit(__load_image, i_fp) for i_fp in i_fps]
        i_img =  [future.result() for future in futures]
        return i_img

def __load_image(i_fp):
    i, filepath = i_fp
    # print(f"load no. {i}: {filepath}")
    return i, cv2.imread(filepath, cv2.IMREAD_COLOR)


from __future__ import annotations

from typing import Any, Optional

import random

import rules
from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
from loguru import logger
from model_utils.models import TimeStampedModel
from rules.contrib.models import RulesModelBase, RulesModelMixin

from .markov_utils import MarkovPOSText
from .rules import (  # is_character_owner,; is_group_owner_and_authenticated,
    is_owner,
    is_owner_or_public,
)
from .signals import markov_sentence_generated, quote_random_retrieved
from .utils import generate_unique_slug_for_model

MAX_QUOTES_FOR_RANDOM_SET = 50
MAX_QUOTES_FOR_RANDOM_GROUP_SET = 50

if hasattr(settings, "MAX_QUOTES_FOR_RANDOM_SET"):  # pragma: nocover
    MAX_QUOTES_FOR_RANDOM_SET = settings.MAX_QUOTES_FOR_RANDOM_SET

if hasattr(settings, "MAX_QUOTES_FOR_RANDOM_GROUP_SET"):  # pragma: nocover
    MAX_QUOTES_FOR_RANDOM_GROUP_SET = settings.MAX_QUOTES_FOR_RANDOM_GROUP_SET

class AbstractOwnerModel(models.Model):
    """
    Abstract model for representing an entity owned by a user with toggles for either allowing submissions for it
    and public access. Defaults to completely private by default.

    Attributes:
        public (bool): is this object public to any authenticated user? Default: False
        allow_submissions (bool): allow other users to submit child objects? Default: False. Not implemented yet.
        owner (User): The user that created this object.
    """

    public = models.BooleanField(
        default=False,
        help_text=_("Is this a public source available for any user to view?"),
    )
    allow_submissions = models.BooleanField(
        default=False, help_text=_("Allow submissions from other users?")
    )
    owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)

    class Meta:
        abstract = True

# Create your models here.
class SourceGroup(
    AbstractOwnerModel, RulesModelMixin, TimeStampedModel, metaclass=RulesModelBase
):
    """
    An abstract group or source for a given set of quotes. Multiple sources, or Source objects, can belong to
    the same group. For example, a novel or series if you plan to quote the characters within individually.

    Attributes:
        id (int): Database primary key for the object.
        name (str): Human readable string to name the group. This will be converted to a slug prefix.
        description (str): A description of the group for convenience. Markdown can be used here for styling.
        description_rendered (str): The HTML representation of the description string. Generated automatically.
        owner (User): The user that created the group and therefore owns it.
        public (bool): Is this group public or private. Defaults to False.
        allow_submissions (bool): Allow other users to submit characters to this. Not yet implemented.
        slug (str): A unique slug to represent this group. Generated automatically from name.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    name = models.CharField(
        _("Source Name"),
        max_length=50,
        help_text=_(
            "A source for individuals making the quotes. Use as an abstract grouping."
        ),
        db_index=True,
    )
    description = models.TextField(
        help_text=_("Description for the source. You can style using Markdown."),
        null=True,
        blank=True,
    )
    description_rendered = models.TextField(
        help_text=_("Automatically generated from description"), null=True, blank=True
    )
    slug = models.SlugField(
        unique=True,
        max_length=70,
        editable=False,
        blank=True,
        help_text=_("Unique slug for this group."),
    )

    @cached_property
    def total_sources(self) -> int:
        return Source.objects.filter(group=self).count()

    @cached_property
    def markov_sources(self) -> int:
        return Source.objects.filter(group=self, allow_markov=True).count()

    @cached_property
    def total_quotes(self) -> int:
        return Quote.objects.filter(
            source__in=Source.objects.filter(group=self)
        ).count()

    @cached_property
    def markov_ready(self) -> bool:
        if (
            self.markov_sources > 0
            and Quote.objects.filter(
                source__in=self.source_set.filter(allow_markov=True)
            ).count()
            > 10
        ):
            return True
        return False

    def generate_markov_sentence(
        self, max_characters: Optional[int] = 280
    ) -> str | None:
        """
        Generate a markov sentence based on quotes from markov enabled characters for the group.

        :return: str or None
        """
        if self.markov_ready:
            logger.debug("Group is ready for markov sentences. Checking model...")
            mmodel = GroupMarkovModel.objects.get(group=self)
            if mmodel.data is None:
                logger.debug(
                    "Markov model for group is not generated yet! Generating..."
                )
                mmodel.generate_model_from_corpus()
            logger.debug("Loading text model...")
            text_model = MarkovPOSText.from_json(mmodel.data)
            logger.debug("Generating sentence...")
            sentence: str | None = text_model.make_short_sentence(
                max_chars=max_characters
            )
            if sentence is not None:
                logger.debug(f"Returning generated sentence: '{sentence}'")
                return sentence
        logger.debug("Group is not ready for markov requests yet!")
        return None

    def get_random_quote(
        self, max_quotes_to_process: Optional[int] = MAX_QUOTES_FOR_RANDOM_GROUP_SET
    ) -> Any:
        """
        Get a random quote object from any of the characters defined within the group.
        Prioritizes quotes that have been returned less often.

        :return: ``Quote`` object or None if no quotes are defined.
        """
        # TODO: Create Q object to filter for null datetimes or less than now.
        quotes = (
            Quote.objects.filter(source__in=self.source_set.all())
            .filter(
                models.Q(pub_date__isnull=True) | models.Q(pub_date__lte=timezone.now())
            )
            .select_related("stats")
            .order_by("stats__times_used")[:max_quotes_to_process]
        )
        if quotes.exists():
            quote = random.choice(list(quotes))
            quote_random_retrieved.send(
                type(quote.source), instance=quote.source, quote_retrieved=quote
            )
            return quote
        return None

    def refresh_from_db(self, *args, **kwargs):
        super().refresh_from_db(*args, **kwargs)
        cached_properties = ["total_sources", "markov_sources", "total_quotes"]
        for prop in cached_properties:
            try:
                del self.__dict__[prop]
            except KeyError:  # pragma: nocover
                pass

    def save(self, *args, **kwargs):
        if (
            not self.slug
        ):  # Once this slug is set, it does not change except through devil pacts
            logger.debug("Group is being saved and a slug was provided.")
            self.slug = generate_unique_slug_for_model(
                model_class=type(self), text=self.name
            )
        super().save(*args, **kwargs)

    def __str__(self):  # pragma: nocover
        return self.name

    class Meta:
        rules_permissions = {
            "add": rules.is_authenticated,
            "read": is_owner_or_public,
            "edit": is_owner,
            "delete": is_owner,
        }
        ordering = ["name"]

class Source(
    AbstractOwnerModel, RulesModelMixin, TimeStampedModel, metaclass=RulesModelBase
):
    """
    An individual source to attribute the quote to in the system, such as a character from a podcast/book, or a specific
    author.

    Attributes:
        id (int): Database primary key for the object.
        name (str): Unique name of a character within a ``CharacterGroup`` for this entity.
        group (SourceGroup): The parent ``SourceGroup``.
        slug (str): Slug made up of a generated version of the character name and the group slug prefix.
        description (str): Description for the character. Markdown can be used for styling.
        description_rendered (str): HTML representation of the description for convenience. Automatically generated.
        allow_markov (bool): Allow markov quotes to be requested from this character? Default False.
        owner (User): The user that created and owns this character.
        public (bool): Is the character public to other users? Defaults to False.
        allow_submissions (bool): Allow other users to submit quotes for this character? Defaults to False.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    name = models.CharField(max_length=100, help_text=_("Name of the character"))
    slug = models.SlugField(
        max_length=250,
        help_text=_(
            "Global slug of the character, will be auto generated from name and group if not overridden."
        ),
        blank=True,
        editable=False,
        unique=True,
        db_index=True,
    )
    description = models.TextField(
        null=True,
        blank=True,
        help_text=_("Description of this character. You can style this with Markdown."),
    )
    description_rendered = models.TextField(
        null=True, blank=True, help_text=_("Automatically generated from description.")
    )
    allow_markov = models.BooleanField(
        default=False, help_text=_("Allow to be used in markov chains?")
    )
    group = models.ForeignKey(
        SourceGroup,
        on_delete=models.CASCADE,
        help_text=_("The group this character belongs to."),
    )

    @property
    def markov_ready(self) -> bool:
        """
        Conducts sanity checks to see if requesting a markov chain is feasible. Markov must be enabled for a character
        and there must be a sufficient corpus to generate a sentence from. Currently set at a minimum of 10 quotes.

        :return: bool
        """
        if self.allow_markov and Quote.objects.filter(source=self).count() > 10:
            return True
        return False

    def get_markov_sentence(self, max_characters: Optional[int] = 280) -> Optional[str]:
        """
        If valid, generate a markov sentence. If not, return None.

        :param max_characters: Optional maximum limit of characters in the return set. Default: 280
        :return: str or None
        """
        logger.debug("Checking to see if character is markov ready...")
        if self.markov_ready:
            logger.debug("It IS ready. Fetching markov model.")
            markov_model = SourceMarkovModel.objects.get(source=self)
            if not markov_model.data:
                logger.debug("No model defined yet, generating...")
                markov_model.generate_model_from_corpus()
            text_model = MarkovPOSText.from_json(markov_model.data)
            logger.debug("Markov text model loaded. Generating sentence.")
            sentence: str | None = text_model.make_short_sentence(
                max_chars=max_characters
            )
            if sentence is not None:
                markov_sentence_generated.send(type(self), instance=self)
                return sentence
        return None

    def get_random_quote(
        self, max_quotes_to_process: Optional[int] = MAX_QUOTES_FOR_RANDOM_SET
    ) -> Optional[Any]:
        """
        This actually not all that random. It's going to grab the quotes
        ordered ordered by how infrequently they've been returned, and then grab a random one
        in the set. But for our purposes, it's fine. If there aren't any quotes, it will return None.

        :return: ``Quote`` object or None
        """
        quotes_to_pick = (
            Quote.objects.filter(source=self)
            .filter(
                models.Q(pub_date__isnull=True) | models.Q(pub_date__lte=timezone.now())
            )
            .select_related("stats")
            .order_by("stats__times_used")[:max_quotes_to_process]
        )
        if quotes_to_pick.exists():
            # Select a random index in the result set.
            quote_to_return = random.choice(list(quotes_to_pick))
            quote_random_retrieved.send(
                type(self), instance=self, quote_retrieved=quote_to_return
            )
            return quote_to_return
        return None

    def __str__(self):  # pragma: nocover
        return self.name

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = generate_unique_slug_for_model(
                type(self), text=f"{self.group.slug} {self.name}"
            )
        super().save(*args, **kwargs)

    class Meta:
        rules_permissions = {
            "add": rules.is_authenticated,
            "read": is_owner_or_public,
            "edit": is_owner,
            "delete": is_owner,
        }

class Quote(
    AbstractOwnerModel, RulesModelMixin, TimeStampedModel, metaclass=RulesModelBase
):
    """
    A quote from a given source.

    Attributes:
        id (int): Database primary key for the object.
        quote (str): The quote text to use. You can use Markdown for styling. Must be <= 280 characters for tweets
        quote_rendered (str): HTML rendered version of the quote field. Automatically generated.
        citation (str): Optional description of quote source, e.g. episode number or book title.
        citation_url (str): Optional accompanying URL for the citation.
        character (Source): The source of this quote.
        owner (User): The user that created and owns this quote.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    quote = models.CharField(
        max_length=280,  # Keep the base limit to 280 so that quotes are 'tweetable'
        help_text="Plain text representation of quote. You can use Markdown here.",
    )
    quote_rendered = models.TextField(
        null=True,
        blank=True,
        help_text=_("HTML rendered version of quote generated from quote plain text."),
    )
    source = models.ForeignKey(
        Source,
        on_delete=models.CASCADE,
        help_text=_("The source of this quote, i.e. name."),
    )
    citation = models.CharField(
        max_length=250,
        blank=True,
        null=True,
        help_text=_("Where is this quote from? Episode #, book?"),
    )
    citation_url = models.URLField(
        null=True, blank=True, help_text=_("URL for citation, if applicable.")
    )
    pub_date = models.DateTimeField(
        null=True,
        blank=True,
        help_text=_("When is the earliest time this should appear in random results?"),
    )

    def __str__(self):  # pragma: nocover
        return f"{self.source.name}: {self.quote}"

    class Meta:
        rules_permissions = {
            # "add": is_character_owner,
            "read": is_owner_or_public,
            "edit": is_owner,
            "delete": is_owner,
        }

class SourceMarkovModel(TimeStampedModel):
    """
    The cached markov model for a given source. The database object for this is automatically created
    whenever a new character object is saved.

    Attributes:
        id (int): Database primary key for the object.
        source (Source): The character who the model is sourced from.
        data (json): The JSON representation of the Markov model created by ``markovify``.
        created (datetime): When this object was first created. Auto-generated.
        modified (datetime): Last time this object was modified. Auto-generated.

    """

    source = models.OneToOneField(Source, on_delete=models.CASCADE)
    data = models.JSONField(null=True, blank=True)

    def generate_model_from_corpus(self):
        """
        Collect all quotes attributed to the related character. Then
        create, compile, and save the model.
        """
        logger.debug("Generating text model. Fetching quotes.")
        quotes = Quote.objects.filter(source=self.source)
        # Don't bother generating model if there isn't data.
        if not quotes.exists():  # pragma: nocover
            logger.debug("There are no quotes. Returning None.")
            return
        logger.debug("Quotes retrieved! Forming into corpus.")
        corpus = " ".join(quote.quote for quote in quotes)
        logger.debug("Building text model.")
        text_model = MarkovPOSText(corpus)
        logger.debug("Compiling text model.")
        text_model.compile(inplace=True)
        logger.debug("Saving model as JSON.")
        self.data = text_model.to_json()
        self.save()
        logger.debug("Markov model populated to database.")

    def __str__(self):  # pragma: nocover
        return self.source.name

class GroupMarkovModel(TimeStampedModel):
    """
    The cached markov model for the entire group. It is made up of every quote from every markov enabled
    character within the group.

    Attributes:
        id (int): The database id of this object.
        group (SourceGroup): The OneToOne relationship to ``SourceGroup``
        data (json): The cached markov model.
        created (datetime): When the object was created.
        modified (datetime): When the object was last modified.
    """

    group = models.OneToOneField(
        SourceGroup,
        on_delete=models.CASCADE,
        help_text=_("The character group this model belongs to."),
    )
    data = models.JSONField(
        null=True, blank=True, help_text=_("The cached markov model as JSON.")
    )

    def generate_model_from_corpus(self):
        """
        Collect all quotes from markov enabled characters in this group and then compile the model and save it.
        """
        logger.debug(f"Gathering corpus for character group: {self.group.name}")
        quotes = Quote.objects.filter(
            source__in=Source.objects.filter(group=self.group, allow_markov=True)
        )
        if quotes.exists():
            if quotes.count() >= 10:
                logger.debug("Found sufficient quotes for a model!")
                corpus = " ".join(quote.quote for quote in quotes)
                logger.debug("Forming text model...")
                text_model = MarkovPOSText(corpus)
                logger.debug("Compiling text model...")
                text_model.compile(inplace=True)
                logger.debug("Saving compiled model to JSON...")
                self.data = text_model.to_json()
                self.save()
                logger.debug(
                    f"Finished building and saving group markov model for {self.group.name}!"
                )

class QuoteStats(TimeStampedModel):
    """
    A simple object used to track how often an individual quote is used.

    Attributes:
        id (int): The database primary key of this object.
        quote (Quote): The quote this stat relates to.
        times_used (int): The number of times this has been used by an service such as random quote.
        created (datetime): When this was created.
        modified (datetime): When this was last modified.
    """

    quote = models.OneToOneField(
        Quote,
        on_delete=models.CASCADE,
        related_name="stats",
        help_text=_("The Quote the stats related to."),
    )
    times_used = models.PositiveIntegerField(
        default=0, help_text=_("Times used for random quotes, etc.")
    )

    def __str__(self):  # pragma: nocover
        return f"Stats for Quote {self.quote.id}"

class GroupStats(TimeStampedModel):
    """
    An object for using to track usage stats for ``CharacterGroup``.

    Attributes:
        group (SourceGroup): The group this is collecting stats for.
        quotes_requested (int): The number of times a quote from this object or its children has been requested.
        quotes_generated (int): The number of times a markov quote has been generated for this or it's children.
    """

    group = models.OneToOneField(
        SourceGroup, related_name="stats", on_delete=models.CASCADE
    )
    quotes_requested = models.PositiveIntegerField(
        default=0, help_text=_("Number of time child quotes have been requested.")
    )
    quotes_generated = models.PositiveIntegerField(
        default=0,
        help_text=_("Number of times markov generated quotes have been requested."),
    )

class SourceStats(TimeStampedModel):
    """
    An object for using to track usage stats for ``Character``.

    Attributes:
        source (Source): The source this is collecting stats for.
        quotes_requested (int): The number of times a quote from this object or its children has been requested.
        quotes_generated (int): The number of times a markov quote has been generated for this or it's children.
    """

    source = models.OneToOneField(
        Source, related_name="stats", on_delete=models.CASCADE
    )
    quotes_requested = models.PositiveIntegerField(
        default=0, help_text=_("Number of time child quotes have been requested.")
    )
    quotes_generated = models.PositiveIntegerField(
        default=0,
        help_text=_("Number of times markov generated quotes have been requested."),
    )


import timeit
import numpy as np

#==============================================================================
class ElectronicSystem(object):
	#--------------------------------------------------------------------------
	def __init__(self, RExPos, REyPos, ETxPos, ETyPos, pumpBeam, stedBeam):
		"""
		Construct a collection of electronic systems, which can consist of
		rare earth ions and electron traps.

		Parameters
		----------
		RExPos : array-like
			Represents the x-coordinates of the rare earths
		REyPos : array-like
			Represents the y-coordinates of the rare earths
		ETxPos : array-like
			Represents the x-coordinates of the electron traps
		ETyPos : array-like
			Represents the y-coordinates of the electron traps
		pumpBeam : Object of type LaserProfiles.PumpBeam()
			Represents the impact of the Gaussian-shaped excitation laser to the electronic systems
		stedBeam : Object of type LaserProfiles.StedBeam()
			Represents the impave of the donut-shaped STED laser
		"""

		if ETxPos.size != ETyPos.size:
			raise ValueError("x and y position array for ET must be of same size.")

		if RExPos.size != REyPos.size:
			raise ValueError("x and y position array for RE must be of same size.")

		# access indices for an electronic system (for both ET and RE)
		self.idx = dict()
		self.idx['x']           = 0
		self.idx['y']           = 1
		self.idx['z']           = 2
		self.idx['isRE']        = 3
		self.idx['isPopulated'] = 4
		self.idx['pDecay']      = 5
		self.idx['pIonize']     = 6
		self.idx['pExcite']     = 7
		self.idx['pRepump']     = 8
		self.idx['pDeplete']    = 9
		self.idx['reState']     = 10
		self.idx['groundStateCounter']  = 11
		self.idx['excitedStateCounter'] = 12

		# encodings for the electronic states in a rare earth
		self._states = dict()
		self._states['ground']  = 1.0
		self._states['excited'] = 2.0
		self._states['ionized'] = 3.0

		# setup and initialize electron traps
		self.electronTraps = np.zeros((ETxPos.size, len(self.idx.keys())))
		self.electronTraps[:, self.idx['x']] = np.array(ETxPos)
		self.electronTraps[:, self.idx['y']] = np.array(ETyPos)
		self.electronTraps[:, self.idx['z']] = np.zeros(ETxPos.size)

		# setup and initialize rare earths
		self.rareEarths = np.zeros((RExPos.size, len(self.idx.keys())))
		self.rareEarths[:, self.idx['x']] = np.array(RExPos)
		self.rareEarths[:, self.idx['y']] = np.array(REyPos)
		self.rareEarths[:, self.idx['z']] = np.zeros(self.rareEarths.shape[0])
		self.rareEarths[:, self.idx['isRE']] = np.ones(self.rareEarths.shape[0])
		self.rareEarths[:, self.idx['isPopulated']] = np.ones(self.rareEarths.shape[0])
		self.rareEarths[:, self.idx['reState']] = np.full(self.rareEarths.shape[0], self._states['ground'])

		# keep laser beams
		self._pumpBeam = pumpBeam
		self._stedBeam = stedBeam

	#--------------------------------------------------------------------------
	def setupTransitionProbabilities(self, gammaRE, sigPumpRE, sigIonizeRE, sigRepumpRE, sigStedRE):
		"""
		Builds and finalizes the collection of electronic systems and calculates
		the transition probabilities of each system depending on its position and
		assigned cross-section.

		Parameters
		----------
		gammaRE : float
			Spontaneous decay of the rare earth (no laser)
		sigPumpRE : float
			Cross-section of the rare earth for excitation (pump laser)
		sigIonizeRE : float
			Cross-section of the rare earth for ionization (pump and STED laser)
		sigRepumpRE : float
			Cross-section of the rare earth for restoration with electron from valence band (pump laser)
		sigStedRE : float
			Cross-section of the rare earth for depletion (STED laser)
		"""

		# calculate pump and STED laser intensities at each electron trap position
		pumpIntensityET = self._pumpBeam.profile(self.electronTraps[:, self.idx['x']], self.electronTraps[:, self.idx['y']])
		stedIntensityET = self._stedBeam.profile(self.electronTraps[:, self.idx['x']], self.electronTraps[:, self.idx['y']])

		# calculate pump and STED laser intensities at each rare earth position
		pumpIntensityRE = self._pumpBeam.profile(self.rareEarths[:, self.idx['x']], self.rareEarths[:, self.idx['y']])
		stedIntensityRE = self._stedBeam.profile(self.rareEarths[:, self.idx['x']], self.rareEarths[:, self.idx['y']])

		# probability for ionizing the electron traps
		pIonizeET = pumpIntensityET + stedIntensityET
		pIonizeET /= np.max(pIonizeET)
		self.electronTraps[:, self.idx['pIonize']] = pIonizeET

		# for rare earth
		probExciteRE  = pumpIntensityRE * sigPumpRE
		probIonizeRE  = (pumpIntensityRE + stedIntensityRE) * sigIonizeRE
		probRepumpRE  = pumpIntensityRE * sigRepumpRE
		probDecayRE   = np.full(self.rareEarths.shape[0], gammaRE)
		probDepleteRE = stedIntensityRE * sigStedRE

		totProbRE = probExciteRE + probIonizeRE + probRepumpRE + probDecayRE + probDepleteRE

		maxTotProbRE   = np.max(totProbRE)
		totProbRE     /= maxTotProbRE
		probExciteRE  /= maxTotProbRE
		probIonizeRE  /= maxTotProbRE
		probRepumpRE  /= maxTotProbRE
		probDecayRE   /= maxTotProbRE
		probDepleteRE /= maxTotProbRE

		#self._probDecayRE   = self._probDecayRE[self._rareEarthIndex]
		probIonizeRE  = probDecayRE  + probIonizeRE
		probExciteRE  = probIonizeRE + probExciteRE
		probRepumpRE  = probExciteRE + probRepumpRE
		probDepleteRE = probRepumpRE + probDepleteRE

		totProbREval   = 10.0 * (probExciteRE + probIonizeRE + probRepumpRE + probDecayRE + probDepleteRE)
		probExciteRE  /= totProbREval
		probIonizeRE  /= totProbREval
		probRepumpRE  /= totProbREval
		probDecayRE   /= totProbREval
		probDepleteRE /= totProbREval

		self.rareEarths[:, self.idx['pDecay']]   = probDecayRE
		self.rareEarths[:, self.idx['pIonize']]  = probIonizeRE
		self.rareEarths[:, self.idx['pExcite']]  = probExciteRE
		self.rareEarths[:, self.idx['pRepump']]  = probRepumpRE
		self.rareEarths[:, self.idx['pDeplete']] = probDepleteRE

		#self._normProbability = pIonizeET
		#self._normProbability[self._rareEarthIndex] = totProbRE[self._rareEarthIndex]

		# concatenate electron traps and rare earths to a single array system
		self.electronicSystem = np.vstack((self.electronTraps, self.rareEarths))

		self.resetRareEarthEvolutionCounters()

	#--------------------------------------------------------------------------
	def createNeighbours(self, electronTravelRange):
		"""creates a collection of neighbour-indices to a certain index
		depending on the given electron travel range."""
		self.neighbours = dict()
		for index in range(self.N):
			indexNeighbours = list()
			pos1 = self.getPosition(index)
			for i in range(self.N):
				pos2 = self.getPosition(i)
				dist = np.linalg.norm(pos1 - pos2)
				if dist <= electronTravelRange:
					indexNeighbours.append(i)
			self.neighbours[index] = np.array(indexNeighbours)

	#--------------------------------------------------------------------------
	def getNeighbours(self, index):
		"""Retruns an array of indices, which are neighbouring to index
		depending on the electron travel range."""
		return self.neighbours[index]

	#--------------------------------------------------------------------------
	def getFreeNeighbours(self, index):
		return np.intersect1d(self.potentialRecombinationIndices, self.getNeighbours(index))

	#--------------------------------------------------------------------------
	@property
	def x(self):
		"""Returns an array of floats, which represents the x coordinates of
		all present electronic systems."""
		return self.electronicSystem[:, self.idx['x']]

	#--------------------------------------------------------------------------
	@property
	def y(self):
		"""Returns an array of floats, which represents the y coordinates of
		all present electronic systems."""
		return self.electronicSystem[:, self.idx['y']]

	#--------------------------------------------------------------------------
	@property
	def z(self):
		"""Returns an array of floats, which represents the z coordinates of
		all present electronic systems."""
		return self.electronicSystem[:, self.idx['z']]

	#--------------------------------------------------------------------------
	@property
	def N(self):
		"""Returns the total number of electronic systems.
		That means electron traps and rare earths."""
		return self.electronicSystem.shape[0]

	#--------------------------------------------------------------------------
	@property
	def rareEarthIndices(self):
		"""Returns an array of integers, which represent the
		indices of the rare earths in the electronic systems."""
		return np.where(self.electronicSystem[:, self.idx['isRE']] == 1.0)[0]

	#--------------------------------------------------------------------------
	@property
	def electronTrapIndices(self):
		"""Returns an array of integers, which represent the
		indices of the electron traps in the electronic systems."""
		return np.where(self.electronicSystem[:, self.idx['isRE']] == 0.0)[0]

	#--------------------------------------------------------------------------
	@property
	def population(self):
		"""Retruns an array of floats, which represents the current
		population of all present electronic systems. A populated system
		is represented by 1.0, a non-populated system is represented by 0.0."""
		return self.electronicSystem[:, self.idx['isPopulated']]

	#--------------------------------------------------------------------------
	def rareEarthGroundStateCounter(self, idx):
		"""Returns a single float, which represents how often the electronic system
		with index idx (which must be a rare earth) has been in ground state since
		the last counter reset."""
		return self.electronicSystem[idx][self.idx['groundStateCounter']]

	#--------------------------------------------------------------------------
	def rareEarthExcitedStateCounter(self, idx):
		"""Returns a single float, which represents how often the electronic system
		with index idx (which must be a rare earth) has been in excited state since
		the last counter reset."""
		return self.electronicSystem[idx][self.idx['excitedStateCounter']]

	#--------------------------------------------------------------------------
	def resetRareEarthEvolutionCounters(self):
		"""Resets both the ground and excited state counter for all rare earths
		in the electronic system."""
		for idx in self.rareEarthIndices:
			self.electronicSystem[idx][self.idx['groundStateCounter']] = 0.0
			self.electronicSystem[idx][self.idx['excitedStateCounter']] = 0.0

	#--------------------------------------------------------------------------
	def recordREstates(self):
		"""Updates either the ground or excited state counter for all rare earths
		present in the system depending on their current electronic states."""
		for idx in self.rareEarthIndices:
			if self.electronicSystem[idx][self.idx['reState']] == self._states['ground']:
				self.electronicSystem[idx][self.idx['groundStateCounter']] += 1.0

			elif self.electronicSystem[idx][self.idx['reState']] == self._states['excited']:
				self.electronicSystem[idx][self.idx['excitedStateCounter']] += 1.0

			else:
				pass

	#--------------------------------------------------------------------------
	def getPosition(self, idx):
		"""Returns an array of floats, which represents the absolute position
		of the electronic system with index idx. The position is read as [x, y, z]."""
		return self.electronicSystem[idx][:self.idx['z'] + 1]

	#--------------------------------------------------------------------------
	def isPopulated(self, idx):
		"""Returns a single float, which indicated whether the electronic system
		with index idx is populated (1.0) or not populated (0.0)."""
		return self.electronicSystem[idx][self.idx['isPopulated']]

	#--------------------------------------------------------------------------
	def populate(self, idx):
		"""Populates the electronic system with index idx. This function
		doesn't take care whether this is physically possible or not, nor
		does it take care if the system to be populated is a rare earth or
		and must be put into some electronic state."""
		self.electronicSystem[idx][self.idx['isPopulated']] = 1.0

	#--------------------------------------------------------------------------
	def isRareEarth(self, idx):
		"""Returns a float representing whether the electronic system
		with index idx is a rare earth (1.0) or an electron trap (0.0)."""
		return self.electronicSystem[idx][self.idx['isRE']]

	#--------------------------------------------------------------------------
	@property
	def potentialRecombinationIndices(self):
		"""Retruns an array of integers representing the indices of all
		electronic systems which are not populated."""
		return np.where(self.electronicSystem[:, self.idx['isPopulated']] == 0.0)[0]

	#--------------------------------------------------------------------------
	def restore(self, idx):
		"""Populates a rare earth with index idx if it is currently ionized
		and puts its electron to ground state. Here, the electron comes from
		the valence band."""
		if self.electronicSystem[idx][self.idx['reState']] == self._states['ionized']:
			self.populate(idx)
			self.electronicSystem[idx][self.idx['reState']] = self._states['ground']
			return 2

		else:
			return 0

	#--------------------------------------------------------------------------
	def excite(self, idx):
		"""Puts the electron of the rare earth with index idx to excited state
		if it currently is in ground state."""
		if self.electronicSystem[idx][self.idx['reState']] == self._states['ground']:
			self.electronicSystem[idx][self.idx['reState']] = self._states['excited']

		return 0

	#--------------------------------------------------------------------------
	def ionizeRE(self, idx):
		"""Ionizes the rare earth with index idx if it currently is in excited state."""
		if self.electronicSystem[idx][self.idx['isPopulated']]:
			if self.electronicSystem[idx][self.idx['reState']] == self._states['excited']:
				self.electronicSystem[idx][self.idx['reState']] = self._states['ionized']
				self.electronicSystem[idx][self.idx['isPopulated']] = 0.0
				return 1

			else:
				return 0

		else:
			return 0

	#--------------------------------------------------------------------------
	def ionizeET(self, idx):
		"""Ionizes an electron trap with index idx."""
		if self.electronicSystem[idx][self.idx['isPopulated']]:
			self.electronicSystem[idx][self.idx['isPopulated']] = 0.0
			return 1

		else:
			return 0

	#--------------------------------------------------------------------------
	def decay(self, idx):
		"""Puts the electron of a rare earth with index idx to ground state
		if it currently is in excited state."""
		if self.electronicSystem[idx][self.idx['reState']] == self._states['excited']:
			self.electronicSystem[idx][self.idx['reState']] = self._states['ground']

		return 0

	#--------------------------------------------------------------------------
	def deplete(self, idx):
		"""Puts the electron of a rare earth with index idx to ground state
		if it currently is in excited state."""
		return self.decay(idx)

	#--------------------------------------------------------------------------
	def recombine(self, idx):
		"""Populates an electronic system with index idx if it currently is not
		populated. If this system is a rare earth, its state is set to excited
		state. Here, the electron comes from the condiction band."""
		self.populate(idx)

		if self.isRareEarth(idx):
			self.electronicSystem[idx][self.idx['reState']] = self._states['excited']

		return 0

	#--------------------------------------------------------------------------
	def actOnElectronTrap(self, idx, probability):
		"""Decides whether an operation is perfomed on an electron trap
		depending on its transition probability."""
		if probability <= self.electronicSystem[idx][self.idx['pIonize']]:
			return self.ionizeET(idx)

		else:
			return 0

	#--------------------------------------------------------------------------
	def actOnRareEarth(self, idx, probability):
		"""Decides whether and which operation is performed on a rare earth
		depending on the corresponding transition probabilities."""
		if probability <= self.electronicSystem[idx][self.idx['pDecay']]:
			return self.decay(idx)

		elif probability <= self.electronicSystem[idx][self.idx['pIonize']]:
			return self.ionizeRE(idx)

		elif probability <= self.electronicSystem[idx][self.idx['pExcite']]:
			return self.excite(idx)

		elif probability <= self.electronicSystem[idx][self.idx['pRepump']]:
			return self.restore(idx)

		elif probability <= self.electronicSystem[idx][self.idx['pDeplete']]:
			return self.deplete(idx)

		else:
			return 0


from google.cloud import storage

import os
from pathlib import Path
import shutil
from time import time
from datetime import datetime

import subprocess
from zipfile import ZipFile,BadZipfile

import requests
import csv

import numpy as np

import logging

def list_gcs_directories(client, bucket, prefix,debug=False):
    # from https://github.com/GoogleCloudPlatform/google-cloud-python/issues/920
    iterator = client.list_blobs(bucket, prefix=prefix, delimiter='/')
    prefixes = set()
    for page in iterator.pages:
        prefixes.update(page.prefixes)
        if debug:
            break #only do one page
    ticker_list = [ticker[5:-1] for ticker in prefixes]
    return ticker_list

def setup_bucket():
    bucket_name = 'kaggle-sec-data'
    storage_client = storage.Client()
    bucket = storage_client.bucket(bucket_name)
    return bucket_name, storage_client, bucket

def get_all_tickers():
    resp = requests.get('https://www.sec.gov/include/ticker.txt')
    ticker_cik_text = resp.content.decode('utf-8')
    ticker_cik_list = list(csv.reader(ticker_cik_text.splitlines(), delimiter='\t'))

    ticker_list = []
    for item in ticker_cik_list:
        ticker_list.append(item[0].upper())

    return ticker_list

debug = True

if debug == True:
    os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="/Users/goldbloom/Dropbox/Side Projects/Edgar/Key/kaggle-playground-0f760ec0ebcd.json"
else:
    os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="./kaggle-playground-0f760ec0ebcd.json"

bucket_name, storage_client, bucket = setup_bucket()

sec_ticker_list = get_all_tickers()

gcs_ticker_list = list_gcs_directories(storage_client,bucket,'data/',False)

complete_ticker_list = []

for ticker in gcs_ticker_list:
    to_match = [f'{ticker} Income Statement.csv', f'{ticker} Balance Sheet.csv', f'{ticker} Cash Flow.csv']
    blobs = storage_client.list_blobs(bucket_name, prefix=f'data/{ticker}/timeseries/Clean Statements/')
    for blob in blobs:
        filename_and_path = blob.name
        filename = filename_and_path[filename_and_path.rfind('/')+1:]
        if filename in to_match:
            to_match.remove(filename)

    if len(to_match) == 0:
        complete_ticker_list.append(ticker)

print("Missing from GCS list")
gcs_missing = [ticker for ticker in sec_ticker_list if ticker not in gcs_ticker_list]
print(gcs_missing)
print("\n")

print("Not complete in GCS")
gcs_not_complete = [ticker for ticker in gcs_ticker_list if ticker not in complete_ticker_list]
print(gcs_not_complete)
print("\n")

print(f"SEC: {len(sec_ticker_list)}; GCS: {len(gcs_ticker_list)}; GCS complete: {len(complete_ticker_list)}")
print(f"Missing from GCS: {len(gcs_missing)}; Not complete in GCS: {len(gcs_not_complete)}")


from datetime import datetime
import argparse
from data import get_dataset

dout_dir = (
    "./out/"
    + datetime.now().strftime("%Y-%m-%d")
    + "/run_"
    + datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
)

default_args = dict(
    epochs=100,
    lr=0.03,
    random_seed=42,
    batch_size = 128,
    save_every = 100,
    out_dir=dout_dir,
    date_time_dir = None,
    exp_name="resnet_test",
    wd = 0.0,
    use_cuda=True,
    det_run=True,
    scale_inv = False,
    loglevel_info=False,
    model = 'resnet18',
    model_width = 64, #default for VGG
    half_prec=False,
    no_data_augm = True,
    batch_norm = False,
    dataset = 'cifar10',
    momentum = 0.9,
    first_decay = 0.5,
    lr_gamma_decay = 1.0,
    lr_flow=1e-4,
    flow_steps=100,
    flow_every = 50,
    wd_flow=0.0,
    wd_gamma_decay = 1.0,
    ckpt = None,
    config_file = None,
    radius = None
)

def configuration(args=None):
    parser = argparse.ArgumentParser(description="Exp Resnet")

    #--------------------------------------------------------------------------------
    # TRAINING OPTIONS
    #--------------------------------------------------------------------------------
    tgroup = parser.add_argument_group("Training options")
    tgroup.add_argument(
        "--epochs",
        type=int,
        metavar="N",
        help="Number of training iters. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--flow_steps",
        type=int,
        metavar="N",
        help="Number of gradient flow iters. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--flow_every",
        type=int,
        metavar="N",
        help="Lr is decayed at multiples of this number. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--batch_size",
        type=int,
        metavar="N",
        help="Batch size. " + "Default: %(default)s.",
    )
    tgroup.add_argument(
        "--lr",
        type=float,
        help="Learning rate of optimizer. Default: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--first_decay",
        type=float,
        help="Fraction of epochs when lr is first decayed: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--lr_gamma_decay",
        type=float,
        help="Decaying factor of the learning rate: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--lr_flow",
        type=float,
        help="Decaying the learning rate to this exact value: " +
        "%(default)s.",
    )
    tgroup.add_argument(
        "--wd_flow",
        type=float,
        help="Decaying the learning rate to this exact value: " +
        "%(default)s.",
    )
    tgroup.add_argument(
        "--wd_gamma_decay",
        type=float,
        help="Decaying factor of the weight decay: " + "%(default)s.",
    )
    tgroup.add_argument(
        "--wd",
        type=float,
        help="Weight decay of optimizer. Default: " + "%(default)s.",
    )

    tgroup.add_argument(
        "--radius",
        type=float,
        help="Radius for PGD. Default: " + "%(default)s.",
    )
    tgroup.add_argument('--momentum', type=float, help="Momentum of optimizer. Default: " + "%(default)s.",)

    #--------------------------------------------------------------------------------
    # MODEL OPTIONS
    #--------------------------------------------------------------------------------
    modelgroup = parser.add_argument_group("Model options")
    modelgroup.add_argument('--model', choices=['vit_basic', 'vit_exp', 'vgg16', 'resnet18', 'resnet18_plain', 'resnet18_gn', 'resnet_tiny', 'resnet_tiny_gn', 'resnet34', 'resnet34_plain', 'resnet34preact', 'resnet34_gn', 'wrn28', 'lenet', 'cnn', 'fc', 'linear'], type=str)
    modelgroup.add_argument('--model_width', type=int, help='model width (# conv filters on the first layer for ResNets)')
    modelgroup.add_argument('--half_prec', action='store_true', help='if enabled, runs everything as half precision [not recommended]')
    modelgroup.add_argument('--batch_norm', action = 'store_true', help = 'if enabled, adds batch-norm layers')
    modelgroup.add_argument('--scale_inv', action = 'store_true', help = 'if enabled, makes the model scale invariant, only for resnets')

    #--------------------------------------------------------------------------------
    # MISCELLANEOUS OPTIONS
    #--------------------------------------------------------------------------------
    mgroup = parser.add_argument_group("Miscellaneous options")
    mgroup.add_argument(
        "--random_seed",
        type=int,
        metavar="N",
        help="Random seed. Default: %(default)s.",
    )
    mgroup.add_argument(
        "--save_every",
        type=int,
        metavar="N",
        help="Numer of iterations for saving model. Default: %(default)s.",
    )
    mgroup.add_argument(
        "--out_dir", type=str, help="Where to store the outputs of this simulation."
    )
    mgroup.add_argument("--exp_name", type=str, help="Name of the experiment.")

    mgroup.add_argument("--date_time_dir", type=str, help="Name of the date time dir.")

    #--------------------------------------------------------------------------------
    # EXPERIMENTS OPTIONS
    #--------------------------------------------------------------------------------
    expgroup = parser.add_argument_group("Experiments options")
    expgroup.add_argument(
        "--use_cuda",
        action="store_true",
        help="Use cuda",
    )
    expgroup.add_argument(
        "--det_run",
        action="store_true",
        help="Ensure deterministic behaviour",
    )
    expgroup.add_argument(
        "--loglevel_info",
        action="store_true",
        help="Info in the logger",
    )
    expgroup.add_argument(
        "--ckpt",
        type=str,
        help="Path for model checkpoint: " + "%(default)s.",
    )
    expgroup.add_argument(
        "--config_file",
        type=str,
        help="Path to specify config file: " + "%(default)s.",
    )
    expgroup.add_argument('--no_data_augm', action='store_true')

    #--------------------------------------------------------------------------------
    # DATA OPTIONS
    #--------------------------------------------------------------------------------
    datagroup = parser.add_argument_group("Dataset options")
    datagroup.add_argument(
        "--input_dim",
        type=int,
        help="Random seed. Default: %(default)s.",
    )
    datagroup.add_argument('--dataset', choices=get_dataset.datasets_dict.keys(), type=str)

    parser.set_defaults(**default_args)

    if args is None:
        return parser.parse_args()
    else:
        return parser.parse_args(args=args)


class Settings:
	# Inicializes using the name of the config file
	def __init__ (self, filename):
		self._filename = filename;

	# Reads a section from the ini
	def LoadSection(self, sectionName):
		# Opens the file
		file = open(self._filename)
		# Loads into memory
		content = file.read()
		content = content.replace('\r', '\n')
		content = content.replace("\n\n", "\n")
		# Close the file
		file.close()
		# Moves up to required section
		startIndex = content.capitalize().find("[" + sectionName.lower() + "]\n")
		if startIndex == -1: raise "ERROR: Section name '" + sectionName + "' count not be found"
		# Find the next one
		endIndex = content.find("\n[", startIndex+1)
		if endIndex == -1: endIndex = len(content)
		# Sets values into a dictionary
		itemsString = content[startIndex + len(sectionName) + 3 : endIndex];
		itemsList = itemsString.splitlines()
		retDictionary = {}
		for item in itemsList:
			item = item.strip()
			if len(item) > 0 and item[0] != '#' :
				pair = item.split('=')
				if len(pair) < 2: raise "ERROR: The line: '" + item + "' is not a valid entry=value item"
				retDictionary[pair[0].strip()] = pair[1].strip()
		# Return the values
		return retDictionary;

	# Reads an entry
	def LoadEntry(self, sectionName, entryName):
		return LoadSection(self, sectionName)[entryName]


# from math import pi

# def AreaOfTheCirce(radius, pi = 3.14):
#     return pi*radius**2

# b = radius=float(input("Enter:"))
# a = pi=float(input("Enter pi:"))
# print(AreaOfTheCirce(b,a))

# def hipo(a, b):
#     c = round((a**2 + b**2)**0.5,1)
#     return c
# print(hipo(2,3))

# if __name__ == "__main__":
#     c,b = 1,5
#     print(hipo(c,b)) #Vlad delal)

# def calc(op, *args):
#     res = 0
#     if op == 'sum':
#         res = 0
#         for el in args:
#            res += el

#     if op == 'dif':
#         for el in args:
#             res -= el
#     return res

# print(calc('dif', 1,2,3,4,5))

def min(*args):
    res = args[0]
    for el in args:
        if el < res:
            res = el
    return res
print(min(23, 45, 34, 4, 9))

num = [3,5,1,9,2,6]
min_num = min(num)
print(min_num)


import requests
import openpyxl
from bs4 import BeautifulSoup

#enter the url here
# url1_indi = "https://www.flipkart.com/dinine-craft-wooden-2-pocket-key-holder-wall-home-decor-decorative-showpiece-6-cm/product-reviews/itmb6135a09b4d7d?pid=SHIGG8J7VQ7BHUYK&lid=LSTSHIGG8J7VQ7BHUYKODYETT&marketplace=FLIPKART"
url1_indi = input("Enter Comments URL Here: ")
page_01 = requests.get(url1_indi)
doc_01 = page_01.text
soup_01 = BeautifulSoup(doc_01,'html.parser')

string = "https://www.flipkart.com"
pages = soup_01.find_all(["a"],class_="ge-49M")

excel = openpyxl.Workbook()
sheet = excel.active
sheet.title = 'Page 1'

insert = []

for i in pages:
    url = string+i['href']
    # print(url)
    page = requests.get(url)
    doc = page.text
    soup = BeautifulSoup(doc,'html.parser')

    for temp in (soup.find_all("div",{ "class" : "col _2wzgFH K0kLPL"})) :
        rating = temp.find("div",{ "class" : "_3LWZlK _1BLPMq"}) #rating
        heading = temp.find("p",{ "class" : "_2-N8zT"}).get_text() #Heading
        comments = temp.find("div",{"class" : "t-ZTKy"}).get_text() #Comments
        comments = comments.removesuffix('READ MORE')
        update = ""
        if rating == None:
            bad_rating = temp.find("div",{ "class" : "_3LWZlK _1rdVr6 _1BLPMq"})
            if bad_rating == None:
                bad_rating = "NULL"
            else:
                bad_rating = bad_rating.get_text()

            update = update + (bad_rating) +"-"
        else:
            update = update + (rating.get_text()) +"-"

        update = update + (heading) +"-"
        update = update + (comments)
        insert.append(update)

sheet.append(insert)

sheet_name = input("Enter the excel sheet file name(No Spaces): ")
excel.save(sheet_name+".xlsx") #change the excel sheet name


## 숫자 자릿수 체크

s= "aabbaccc"

lenList= []

for i in range(1, int(len(s)/2)+1): #문자열 자르는 단위
    ind= 0
    l= 0
    count= 1
    d=0

    for j in range(len(s)): #전체 문자열 검사
        if ind+ i> len(s): #현재 인덱스범위가 전체 길이를 벗어나면 반복문 탈출
            print(ind, ind+2*i)
            break
        if s[ind:ind+i]== s[ind+i:ind+ 2*i]:
            l+=i
            count+= 1

            # print(len(s), l)
        elif count>1:
            d+= len(str(count)) #숫자의 자릿수 정보
            count= 1
        ind+=i
    lenList.append(len(s)-l+d)
    #print(lenList)

print(min(lenList))


# 导入tushare
import tushare as ts

# 初始化pro接口
pro = ts.pro_api('ca0af3044cc38461f8e4ae128c9edabc12bcab9f4628f5cf6b6d863a')

# 拉取数据
df = pro.index_dailybasic(**{
    "trade_date": "",
    "ts_code": "000001.SH",
    "start_date": "",
    "end_date": "",
    "limit": "",
    "offset": ""
}, fields=[
    "ts_code",
    "trade_date",
    "total_mv",
    "float_mv",
    "total_share",
    "float_share",
    "free_share",
    "turnover_rate",
    "turnover_rate_f",
    "pe",
    "pe_ttm",
    "pb"
])
print(df)

05-指数/06-大盘指数每日指标.py

06-大盘指数每日指标.py


from pytube import YouTube, Playlist
from pytube.cli import on_progress
import os

# VIDEO_URL = 'https://www.youtube.com/watch?v=D26bLJ9ut88'
# PLAYLIST_URL = 'https://www.youtube.com/watch?v=Ck3jX3wbyfY&list=PLmHzN0jU_yECKq_7W0DASIGejHHrMgKw0&ab_channel=CortesdoLutz%5BOFICIAL%5D'

while True:
    #Limpando console
    if os.name == "nt":
        os.system("cls")
    else:
        os.system("clear")

    #Escolher tipo de download
    answer = int(input("Deseja fazer download de um video ou playlist ? \n [1]Vídeo\n [2]Playlist\n Resposta:"))

    if answer == 1:
        url_video = input("Digite a url do vídeo: ")
        yt = YouTube(url_video, on_progress_callback=on_progress)
        yt.streams.get_highest_resolution().download()
        break
    elif answer == 2:
        url_playlist = input("Digite a url da playlist: ")
        playlist = Playlist(url_playlist)

        for video in playlist:
            yt = YouTube(video, on_progress_callback=on_progress)
            yt.streams.get_highest_resolution().download()
        break

print("Download complete")


import sys
sys.stdin = open('h.txt', 'r')

import sys
sys.setrecursionlimit(99999999)

from collections import deque
def bfs(a):
    visit[a] = True
    q = deque()
    q.append(a)
    while q:
        n = q.popleft()
        for i in range(len(G[n])):
            if visit[G[n][i]]:continue
            visit[G[n][i]] = True
            q.append(G[n][i])

def dfs(a):
    visit[a] = True
    for i in range(len(G[a])):
        if visit[G[a][i]]:continue
        dfs(G[a][i])

N, M = map(int, input().split())
arr = [list(map(int, input().split())) for _ in range(M)]
G = [[] for _ in range(N+1)]
visit = [False] * (N+1)
for i in range(M):
    G[arr[i][1]].append(arr[i][0])
for i in range(1, len(visit)):
    if visit[i]: continue
    dfs(i)
    print(i,end=' ')
print()


# Find Connected Components
# By Carson Forsyth
# From Professor Mingfu Shao
# CMPSCI 465 @ PSU, FA2020

# This dynamic programming algorithm will find a maximum weight subset of mutually compatible jobs.
# Takes in n jobs: job i starts from S_i, finishes at T_i, and has the weight V_i.

import math

# Take in an array of lists [start, terminate, weight] that is len_array long.
# Return optimal weight achievable considering all jobs.

def dp_scheduling(array, len_array):
    array.sort(key=lambda elem: int(elem[1]))
    opt = [0]*len_array
    pre = [0]*len_array
    opt[0] = array[0][2]
    for k in range(1, len_array):
        pre[k] = binary_search(array, k, 0, k - 1)
        if array[k][0] < array[pre[k]][1]:
            opt[k] = max(opt[k-1], (array[k][2]))
        else:
            opt[k] = max(opt[k-1], (array[k][2]+opt[pre[k]]))
    return opt[len_array-1]

# Take in sorted array of jobs. Find largest element with termination time that is compatible with job k.

def binary_search(array, k, a, b):
    m = math.floor((a + b) / 2)
    if b <= a:
        return a
    elif array[m][1] <= array[k][0] < array[m + 1][1]:
        return m
    elif array[m][1] > array[k][0]:  # Termination of m is greater than start of k: check array prior to m.
        return binary_search(array, k, a, m)
    elif array[k][0] >= array[m+1][1]:  # Start of k is after termination of the array after m: check
        return binary_search(array, k, m+1, b)
    else:
        return -1

num_jobs = int(input())
jobs = []
for i in range(num_jobs):
    jobs.append([int(x) for x in input().split()])
print(dp_scheduling(jobs, num_jobs))


class Solution:
    def findJudge(self, n: int, trust: List[List[int]]) -> int:
        # use two hash tables
        # This is the idea:
        # 1) Scan thru the array "trust" and record two things in two separate
        # hash tables:
        #   - the number of outgoing edges from the node in current relationship (a_i)
        #   - the number of incoming edges from the node in current relationship node (b_i)
        # Then, scan thru the outgoing edges hash table and look for the
        # node that has ZERO outgoing edges and (N-1) incoming edges, and
        # return that node.
        # Otherwise you will scan thru the entire hash table and not find a
        # accompanying match, so return a -1.
        #
        # Runtime: O(N) b/c every single person in the town is added into both the
        # incoming and outgoing edges hashmaps. Then you scan thru the entire
        # town in the outgoing hashmap and cross-check with the incoming hashmap
        # to look for the existence of the town judge.
        # Space used: O(2N) --> O(N) b/c two hashmaps are used in the process.

        num_outgoing_edges = dict()
        num_incoming_edges = dict()

        for i in range(1, n + 1):
            num_outgoing_edges[i] = 0
            num_incoming_edges[i] = 0

        for relationship in trust:
            outgoing_edge = relationship[0]
            incoming_edge = relationship[1]

            if outgoing_edge in num_outgoing_edges.keys():
                num_outgoing_edges[outgoing_edge] += 1
            else:
                num_outgoing_edges[outgoing_edge] = 1

            if incoming_edge in num_incoming_edges.keys():
                num_incoming_edges[incoming_edge] += 1
            else:
                num_incoming_edges[incoming_edge] = 1

        for edge in num_outgoing_edges.keys():
            if num_outgoing_edges[edge] == 0 and\
            num_incoming_edges[edge] == n - 1:
                return edge
        return -1

    # class Solution: ---> Improved solution
    # def findJudge(self, n: int, trust: List[List[int]]) -> int:
    #     # think like a graph
    #     # use two dicts
    #     # first dict --> for each node in graph, stores number of outgoing edges
    #     # second dict --> for each node in graph, stores number of incoming edges
    #     # -> town judge: the node has zero outgoing edges and also has
    #     # (N - 1) incoming edges
    #     #
    #     # 1) loop thru trust array and count number of outgoing edges for
    #     # each node (a_i)
    #     # 2) loop thru trust array and count number of incmoing edges for
    #     # each node (b_i)
    #     # 3) loop thru the first dict and find a match in the second dict
    #     # if exists, else return -1

    #     # loop thru 'n' and populate the both dictionaries with key
    #     # 'n' value and value initialized 0
    #     # Runtime: O(V + E) b/c all vertices looked at to populate the dictionaries
    #     # and all directed edges looked at to count up each vertex's total
    #     # num of incoming and outgoing edges.
    #     # Space used: O(V) where dictionary sizes are proportional to total num
    #     # of vertices in the graph. (V key:value pairs in each dict.)

    #     incoming_edges = dict()
    #     outgoing_edges = dict()

    #     for i in range(1, n + 1):
    #         incoming_edges[i] = 0
    #         outgoing_edges[i] = 0

    #     for relationship in trust:
    #         outgoing_edges[relationship[0]] += 1
    #         incoming_edges[relationship[1]] += 1

    #     for node in incoming_edges.keys():
    #         if incoming_edges[node] == n - 1 and outgoing_edges[node] == 0:
    #             return node
    #     return -1


import os, time, datetime
from flask import flash, render_template, redirect, send_file, request, url_for
from flask_login import current_user, login_required, user_unauthorized
from google.cloud import storage
from app.admin.forms import CommentForm, UserForm
from app import db
from app.models import User, Image, Comment
from app.admin.forms import ImageForm
from app.main import bp
from config import basedir, GOOGLE_AUTH, BUCKET_NAME

@bp.route('/')
@bp.route('/index')
@login_required
def index():
    if not user_unauthorized:
        if current_user.is_admin:
            return redirect('admin.index')
    images = Image.query.filter_by(user_id= current_user.id).all()
    if len(images) > 0:
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))

        bucket = gcs.bucket(BUCKET_NAME)

        for i in images:
            bolb = bucket.blob(i.src)
            bolb.make_public()
            i.file_path = bolb.public_url

    return render_template('index.html', user= current_user, images = images)

@bp.route('/image/<int:id>', methods=['GET'])
def image(id):
    image = Image.query.filter_by(id= id, user_id = current_user.id).one()
    if image:
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))

        bucket = gcs.bucket(BUCKET_NAME)
        bolb = bucket.blob(image.src)
        image.file_path = bolb.public_url
    comments = Comment.query.filter_by(image_id=id).all()

    form = CommentForm(csrf_enabled= False)

    return render_template('main/image.html', user= current_user, image= image, comments= comments, form= form, action_url=url_for('main.add_comment', imgid= image.id), btn_label='Add Comment')

@bp.route('/images', methods= ['GET'])
@login_required
def images():
    images = Image.query.filter_by(user_id = current_user.id).all()
    if len(images) > 0:
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))

        bucket = gcs.bucket(BUCKET_NAME)

        for i in images:
            bolb = bucket.blob(i.src)
            i.file_path = bolb.public_url
    return render_template('main/images.html', user= current_user, images= images)

@bp.route('/image/add', methods=['GET', 'POST'])
@login_required
def add_image():
    form = ImageForm(csrf_enabled= False)
    if form.validate_on_submit() and request.method == 'POST':
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
        bucket = gcs.bucket(BUCKET_NAME)

        image_data = form.image.data.stream.read()

        now = str(int(time.time()))
        n_filename = now + '_' + form.image.data.filename

        blob = bucket.blob(n_filename)

        blob.upload_from_string(image_data, content_type=form.image.data.mimetype)
        blob.make_public()
        image = Image(title= form.title.data, src= n_filename , user_id= current_user.id, description= form.description.data)
        db.session.add(image)
        db.session.commit()

        return redirect(url_for('main.images'))
    return render_template('main/add_image.html', user= current_user, form= form, action_url=url_for('main.add_image'), btn_label='Add Image')

@bp.route('/image/edit/<int:id>', methods=['GET', 'POST'])
@login_required
def edit_image(id):
    image = Image.query.filter_by(id= id, user_id = current_user.id).one()
    form  = ImageForm(csrf_enabled= False, obj=image)
    if form.validate_on_submit() and request.method == 'POST':
        gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
        bucket = gcs.bucket(BUCKET_NAME)

        image_data = form.image.data.stream.read()

        now = str(int(time.time()))
        n_filename = now + '_' + form.image.data.filename

        blob = bucket.blob(n_filename)
        o_blob = bucket.blob(image.src)
        o_blob.delete()

        blob.upload_from_string(image_data, content_type=form.image.data.mimetype)
        blob.make_public()
        image.src = n_filename
        image.title = form.title.data

        image.description = form.description.data
        db.session.merge(image)
        db.session.commit()

        return redirect(url_for('main.images'))
    return render_template('main/add_image.html', user= current_user, form= form, action_url=url_for('main.edit_image', id= id), btn_label="Update Image")

@bp.route('/image/delete/<int:id>', methods=['GET'])
@login_required
def delete_image(id):
    image = Image.query.filter_by(id = id, user_id= current_user.id).one()
    gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
    bucket = gcs.bucket(BUCKET_NAME)
    blob = bucket.blob(image.src)
    blob.delete()
    db.session.delete(image)
    db.session.commit()
    # flash('Image Deleted!')
    return redirect(url_for('main.images'))

@bp.route('/image/download/<int:id>', methods=['GET'])
@login_required
def download_image(id):
    image = Image.query.filter_by(id = id).one()
    gcs = storage.Client.from_service_account_json(os.path.join(basedir, GOOGLE_AUTH))
    bucket = gcs.bucket(BUCKET_NAME)
    blob = bucket.blob(image.src)
    temppath = os.path.join(basedir, 'app/static/tempimg/')
    if not os.path.exists(temppath):
        os.mkdir(temppath)
    with open(os.path.join(temppath, blob.name), 'wb') as file:
        blob.download_to_file(file)
    return send_file(os.path.join(temppath, blob.name), attachment_filename=blob.name, mimetype='image/jpg')

@bp.route('/user/edit', methods=['GET', 'POST'])
@login_required
def edit_user():
    user = User.query.filter_by(id= current_user.id).one()
    form = UserForm(csrf_enabled= False, obj= user)
    if form.validate_on_submit() and request.method == 'POST':
        user.username = form.username.data
        user.set_password(form.username.data)
        user.address = form.address.data
        user.email = form.email.data

        db.session.merge(user)
        db.session.commit()

        # flash('User profile updated!')
        return redirect(url_for('main.index'))
    return render_template('main/edit_user.html',user= current_user, form= form, action_url=url_for('main.edit_user'), btn_label="Update User Profile")

@bp.route('/comment/add/<int:imgid>', methods=['POST'])
def add_comment(imgid):
    form = CommentForm(csrf_enabled= False)
    if form.validate_on_submit() and request.method == 'POST':
        comment = Comment(content= form.content.data, image_id=imgid, user_id= current_user.id, created_at= datetime.datetime.now())
        db.session.add(comment)
        db.session.commit()
        # flash('Comment Added')
    return redirect(url_for('main.image', id= imgid))


import re
from typing_extensions import Self
from flask import Flask, request, template_rendered, Blueprint
from flask import url_for, redirect, flash
from flask import render_template
from flask_login import LoginManager, UserMixin, login_user, logout_user, login_required, current_user
from datetime import datetime
from numpy import identity, product
import random, string
from sqlalchemy import null
from link import *
import math
from base64 import b64encode
from api.sql import *

store = Blueprint('bookstore', __name__, template_folder='../templates')

@store.route('/', methods=['GET', 'POST'])
@login_required
def bookstore():
    result = Product.count()
    count = math.ceil(result[0]/6)
    flag = 0

    if request.method == 'GET':
        if(current_user.role == 'manager'):
            flash('No permission')
            return redirect(url_for('manager.home'))

    if 'keyword' in request.args and 'page' in request.args:
        total = 0
        single = 1
        page = int(request.args['page'])
        start = (page - 1) * 6
        end = page * 6
        search = request.values.get('keyword')
        keyword = search

        cursor.prepare('SELECT * FROM PRODUCT WHERE PNAME LIKE :search')
        cursor.execute(None, {'search': '%' + keyword + '%'})
        book_row = cursor.fetchall()
        book_data = []
        final_data = []

        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5],
            }
            book_data.append(book)
            total = total + 1

        if(len(book_data) < end):
            end = len(book_data)
            flag = 1

        for j in range(start, end):
            final_data.append(book_data[j])

        count = math.ceil(total/6)

        return render_template('bookstore.html', single=single, keyword=search, book_data=book_data, user=current_user.name, page=1, flag=flag, count=count)

    elif 'pid' in request.args:
        pid = request.args['pid']
        data = Product.get_product(pid)

        pname = data[1]
        price = data[2]
        sname = data[3]
        description = data[4]

        image = 'default.png'  #在此更改圖片
        if(data[5] != None):
            image = data[5]

        sdesc = '尚未新增品牌介紹'
        if(SUPPLIER.get_sDesc(sname) != None):
            sdesc = SUPPLIER.get_sDesc(sname)[0]

        product = {
            '商品編號': pid,
            '商品名稱': pname,
            '單價': price,
            '品牌': sname,
            '商品敘述': description,
            '商品圖片': image,
            '品牌介紹': sdesc
        }

        return render_template('product.html', data = product, user=current_user.name)

    elif 'page' in request.args:
        page = int(request.args['page'])
        start = (page - 1) * 6
        end = page * 6

        book_row = Product.get_all_product()
        book_data = []
        final_data = []

        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5]
            }
            book_data.append(book)

        if(len(book_data) < end):
            end = len(book_data)
            flag = 1

        for j in range(start, end):
            final_data.append(book_data[j])

        return render_template('bookstore.html', book_data=final_data, user=current_user.name, page=page, flag=flag, count=count)

    elif 'keyword' in request.args:
        single = 1
        search = request.values.get('keyword')
        keyword = search
        cursor.prepare('SELECT * FROM PRODUCT WHERE PNAME LIKE :search')
        cursor.execute(None, {'search': '%' + keyword + '%'})
        book_row = cursor.fetchall()
        book_data = []
        total = 0

        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5]
            }

            book_data.append(book)
            total = total + 1

        if(len(book_data) < 6):
            flag = 1

        count = math.ceil(total/6)

        return render_template('bookstore.html', keyword=search, single=single, book_data=book_data, user=current_user.name, page=1, flag=flag, count=count)

    else:
        book_row = Product.get_all_product()
        book_data = []
        temp = 0
        for i in book_row:
            book = {
                '商品編號': i[0],
                '商品名稱': i[1],
                '商品價格': i[2],
                '商品圖片': i[5]
            }
            if len(book_data) < 6:
                book_data.append(book)

        return render_template('bookstore.html', book_data=book_data, user=current_user.name, page=1, flag=flag, count=count)

# 會員購物車
@store.route('/cart', methods=['GET', 'POST'])
@login_required # 使用者登入後才可以看
def cart():

    # 以防管理者誤闖
    if request.method == 'GET':
        if( current_user.role == 'manager'):
            flash('No permission')
            return redirect(url_for('manager.home'))

    # 回傳有 pid 代表要 加商品
    if request.method == 'POST':

        if "pid" in request.form :
            data = Cart.get_cart(current_user.id)

            if( data == None): #假如購物車裡面沒有他的資料
                time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                Cart.add_cart(current_user.id, time) # 幫他加一台購物車
                data = Cart.get_cart(current_user.id)

            cid = data[3] # 取得購物車編號(cId)
            pid = request.values.get('pid') # 使用者想要購買的東西
            # 檢查購物車裡面有沒有商品
            product = CONTAIN.check_product(cid, pid)
            # 取得商品價錢
            price = Product.get_product(pid)[2]

            # 如果購物車裡面沒有的話 把他加一個進去
            if(product == None):
                CONTAIN.add_product( {'cid': cid, 'pid':pid} )
            else:
                # 假如購物車裡面有的話，就多加一個進去
                amount = CONTAIN.get_amount(cid, pid)
                CONTAIN.update_product({'amount':amount+1, 'pid':pid, 'cid':cid})

        elif "delete" in request.form :
            pid = request.values.get('delete')
            cid = Cart.get_cart(current_user.id)[3]

            Member.delete_product(cid, pid)
            product_data = only_cart()

        elif "user_edit" in request.form:
            change_order()
            return redirect(url_for('bookstore.bookstore'))

        elif "buy" in request.form:
            change_order()
            return redirect(url_for('bookstore.order'))

        elif "order" in request.form:
            cid = Cart.get_cart(current_user.id)[3]
            total = CONTAIN.get_total(cid)

            time = str(datetime.now().strftime('%Y/%m/%d %H:%M:%S'))
            format = 'yyyy/mm/dd hh24:mi:ss'
            TRANSACTION.add_order( {'mid': current_user.id,
                                    'time':time,
                                    'cid':cid,
                                    'format':format,
                                    'payment':'轉帳',
                                    } )

            tno = TRANSACTION.get_tno(cid)[0][0]
            Cart.set_cart(cid, tno)

            Record.add_record(cid, tno)

            return render_template('complete.html', user=current_user.name)

    product_data = only_cart()

    if product_data == 0:
        return render_template('empty.html', user=current_user.name)
    else:
        return render_template('cart.html', data=product_data, user=current_user.name)

@store.route('/order')
def order():
    data = Cart.get_cart(current_user.id)
    cid = data[3]

    product_row = CONTAIN.get_record(cid)
    product_data = []

    for i in product_row:
        pname = Product.get_name(i[2])
        product = {
            '商品編號': i[2],
            '商品名稱': pname,
            '商品價格': Product.get_price(i[2]),
            '數量': i[1]
        }
        product_data.append(product)

    total = CONTAIN.get_total(cid)

    return render_template('order.html', data=product_data, total=total, user=current_user.name)

@store.route('/orderlist', methods=['GET', 'POST'])
def orderlist():
    #新增取消訂單的功能
    if 'delete' in request.values:
        tno = request.values.get('delete')
        data = TRANSACTION.delete_order(tno)

    if "oid" in request.args :
        pass

    user_id = current_user.id

    data = Member.get_order(user_id)
    orderlist = []

    for i in data:
        temp = {
            '訂單編號': i[3],
            '訂單總價': CONTAIN.get_total(i[2]),
            '訂單時間': i[1]
        }
        orderlist.append(temp)

    orderdetail_row = TRANSACTION.get_orderdetail()
    orderdetail = []

    for j in orderdetail_row:
        temp = {
            '訂單編號': j[0],
            '商品名稱': j[1],
            '商品單價': j[2],
            '訂購數量': j[3]
        }
        orderdetail.append(temp)

    return render_template('orderlist.html', data=orderlist, detail=orderdetail, user=current_user.name)

def change_order():
    data = Cart.get_cart(current_user.id)
    cid = data[3] # 使用者有購物車了，取得購物車編號
    product_row = CONTAIN.get_record(cid)

    for i in product_row:

        # i[0]：交易編號 / i[1]：商品編號 / i[2]：數量 / i[3]：價格
        # i[0]：cid / i[1]：商品數量 / i[2]：pid
        if int(request.form[i[2]]) != i[1]:
            CONTAIN.update_product({
                'amount':request.form[i[2]],
                'pid':i[2],
                'cid':cid,
                #'total':int(request.form[i[1]])*int(Product.get_price(i[2]))
            })
            print('change')

    return 0

def only_cart():

    count = Cart.check(current_user.id)

    if(count == None):
        return 0

    data = Cart.get_cart(current_user.id)
    cid = data[3]
    product_row = CONTAIN.get_record(cid)
    product_data = []

    for i in product_row:
        pid = i[2]
        pname = Product.get_name(pid)
        price = Product.get_price(pid)
        amount = i[1]

        product = {
            '商品編號': pid,
            '商品名稱': pname,
            '商品價格': price,
            '數量': amount
        }
        product_data.append(product)

    return product_data


import matplotlib.pyplot as plt
import pandas as pd

from q51 import load_data
from q52 import create_model

train_x_vec = pd.read_table('files/train.feature.txt', header=None)
valid_x_vec = pd.read_table('files/valid.feature.txt', header=None)
test_x_vec = pd.read_table('files/test.feature.txt', header=None)

_, train_y = load_data('files/train.txt')
_, valid_y = load_data('files/valid.txt')
_, test_y = load_data('files/test.txt')

for index, penalty in enumerate(['l1', 'l2'], start=1):
    models = [
        create_model(train_x_vec, train_y, C=10**(-c), random_state=42, penalty=penalty)
        for c in range(-2, 6)
    ]
    plt.subplot(2, 1, index)
    plt.xscale('log')
    plt.title(f'{penalty.upper()}')
    plt.plot(
        [10**(-c) for c in range(-2, 6)],
        [model.score(train_x_vec, train_y) for model in models],
        label='train',
        marker='o'
    )
    plt.plot(
        [10**(-c) for c in range(-2, 6)],
        [model.score(valid_x_vec, valid_y) for model in models],
        label='valid',
        marker='o'
    )
    plt.plot(
        [10**(-c) for c in range(-2, 6)],
        [model.score(test_x_vec, test_y) for model in models],
        label='test',
        marker='o'
    )
    plt.ylim([0, 1])
    plt.legend()

plt.show()


#Part 1
"""

Opening the front panel reveals a jumble of wires. Specifically, two wires are connected to a central port and extend outward on a grid.
You trace the path each wire takes as it leaves the central port, one wire per line of text (your puzzle input).

The wires twist and turn, but the two wires occasionally cross paths.
To fix the circuit, you need to find the intersection point closest to the central port.
Because the wires are on a grid, use the Manhattan distance for this measurement.
While the wires do technically cross right at the central port where they both start,
this point does not count, nor does a wire count as crossing with itself.
Here are a few more examples:

R75,D30,R83,U83,L12,D49,R71,U7,L72
U62,R66,U55,R34,D71,R55,D58,R83 = distance 159

R98,U47,R26,D63,R33,U87,L62,D20,R33,U53,R51
U98,R91,D20,R16,D67,R40,U7,R15,U6,R7 = distance 135

What is the Manhattan distance from the central port to the closest intersection?
"""

def main():
    fileinput = input("Insert full path of input file (Ex: c:/Documents/Advent/Day3/inputfile.txt):")
    f = open(fileinput, "r")
    lst = f.readlines()
    lst = formatinput(lst)
    part1 = abssum(list(filtercolisions(list(setpath(lst)))))
    print("Part 1: ", part1)
    part2 = abssumcol(list(pathcolisions(list(setpath(lst)))))
    print("Part 2:", part2)

def pathcolisions(lst):
    colisions = list(filtercolisions(lst))
    pathcol = []
    for x in colisions:
        for y in lst:
            path = []
            for z in y:
                if z != x:
                    path.append(z)
                else:
                    path.append(z)
                    break
            pathcol.append(path)

    return pathcol

def abssum(lst):
    smallest = abs(lst[0][0]) + abs(lst[0][1])
    for x in lst:
        if abs(x[0]) + abs(x[1]) < smallest:
            smallest = abs(x[0]) + abs(x[1])
    return smallest

def abssumcol(lst):
    smallest = len(list(lst[0])) + len(list(lst[1]))
    for x in range(len(lst)):
        if x % 2 == 0:
            if len(list(lst[x])) + len(list(lst[x+1])) < smallest:
                smallest = len(list(lst[x])) + len(list(lst[x+1]))

    return smallest

def filtercolisions(lsttest2):
    filterlist = []
    for x in lsttest2[0]:
        if x in lsttest2[1]:
            filterlist.append(x)
    return filterlist

def formatinput(lst):
    lst2 = []
    for x in lst:
        lst2.append(list(x.strip('\n').split(',')))
    return lst2

def setpath(lst):
    pathlist = []
    for x in lst:
        center = (0,0)
        points = []
        for y in x:
            if y[0] == "R":
                for i in range(int(y[1:])):
                    center = (center[0] + 1, center[1])
                    points.append(center)
            elif y[0] == "L":
                for i in range(int(y[1:])):
                    center = (center[0] - 1, center[1])
                    points.append(center)
            elif y[0] == "U":
                for i in range(int(y[1:])):
                    center = (center[0], center[1] + 1)
                    points.append(center)
            elif y[0] == "D":
                for i in range(int(y[1:])):
                    center = (center[0], center[1] - 1)
                    points.append(center)
            else:
                raise Exception("Invalid direction!!!!!!!!")
        pathlist.append(list(points))
    return pathlist

main()


# 진수 변환기(16진수는 자릿수가 10미만인 것만 사용 가능)

print("변환하고자 하는 진수법 2개를 입력해주세요 (2진수, 8진수, 10진수, 16진수)")
print("예) 10 2 (10진수 -> 2진수)")

from_, to_ = map(int, input().split())

print("변환하고자 하는 값을 입력하세요")
# if from_ == 16:
#     val = list(input())
# else:
val = int(input())

binary = ''
octal = ''
decimal = 0
hexadecimal = ''

hex_bi = {10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F'}
bi_hex = {'A': 10, 'B': 11, 'C': 12, 'D': 13, 'E': 14, 'F': 15}

def binary_number(to_, val):
    global decimal
    global octal
    global hexadecimal
    global hex_bi
    tmp_arr = []

    if to_ == 8:
        # 2의 0승부터 3자리씩 잘라서 더해주는 방법
        while val != 0:
            bi = 1
            tmp = 0

            for _ in range(3):
                if val == 0:
                    break
                tmp += (bi * (val % 10))
                val //= 10
                bi *= 2
            tmp_arr.append(str(tmp))
        # 뒤에서부터 자릿수 만들기
        for num in tmp_arr[::-1]:
            octal += num
        return int(octal)

    elif to_ == 10:
        bi = 1
        # 2의 0승 자리부터 잘라서 더해주는 방법
        while val != 0:
            decimal += (bi * (val % 10))
            val //= 10
            bi *= 2
        return decimal

    # to_ == 16
    else:
        # 2의 0승부터 4자리씩 잘라서 더해주는 방법
        while val != 0:
            bi = 1
            tmp = 0

            for _ in range(4):
                if val == 0:
                    break
                tmp += (bi * (val % 10))
                val //= 10
                bi *= 2
            # 10보다 작은 수는 그대로 문자로 바꿔서 넣기
            if tmp < 10:
                tmp_arr.append(str(tmp))
            # 크면 딕셔너리에서 찾아서 넣기
            else:
                tmp_arr.append(hex_bi[tmp])
        # 뒤에서부터 자릿수 만들기
        for num in tmp_arr[::-1]:
            hexadecimal += num
        return hexadecimal

def octal_number(to_, val):
    global binary
    tmp_arr = []

    if to_ == 2:
        while val != 0:
            tmp = val % 10

            while tmp != 0:
                if tmp % 2 == 0:
                    tmp_arr.append('0')
                else:
                    tmp_arr.append('1')
                tmp //= 2
            val //= 10

            # 3자리 단위로 출력을 위해 0이부족하면 추가
            if len(tmp_arr) % 3 != 0:
                while len(tmp_arr) % 3 != 0:
                    tmp_arr.append('0')

        # 뒤에서부터 자릿수 만들기
        for num in tmp_arr[::-1]:
            binary += num
        return int(binary)

    elif to_ == 10:
        return binary_number(10, octal_number(2, val))

    # to_ == 16
    else:
        return binary_number(16, octal_number(2, val))

def decimal_number(to_, val):
    global binary
    tmp_arr = []

    if to_ == 2:
        while val != 0:
            if val % 2 == 0:
                tmp_arr.append('0')
            else:
                tmp_arr.append('1')
            val //= 2
        # 뒤에서부터 자릿수 만들기
        for num in tmp_arr[::-1]:
            binary += num
        return int(binary)

    elif to_ == 8:
        return binary_number(8, decimal_number(2, val))

    # to_ == 16
    else:
        return binary_number(16, decimal_number(2, val))

def hexadecimal_number(to_, val):
    global binary
    global bi_hex
    tmp_arr = []

    if to_ == 2:
        while val != 0:
            tmp = val % 10

            if tmp in bi_hex:
                tmp = int(bi_hex[tmp])

            while tmp != 0:
                if tmp % 2 == 0:
                    tmp_arr.append('0')
                else:
                    tmp_arr.append('1')
                tmp //= 2
            val //= 10

            # 4자리 단위로 출력을 위해 0이부족하면 추가
            if len(tmp_arr) % 4 != 0:
                while len(tmp_arr) % 4 != 0:
                    tmp_arr.append('0')

        # 뒤에서부터 자릿수 만들기
        for num in tmp_arr[::-1]:
            binary += num
        return int(binary)

    elif to_ == 8:
        return binary_number(8, hexadecimal_number(2, val))

    # to_ == 10
    else:
        return binary_number(10, hexadecimal_number(2, val))

if from_ == 2:
    result = binary_number(to_, val)
elif from_ == 8:
    result = octal_number(to_, val)
elif from_ == 10:
    result = decimal_number(to_, val)
else:
    result = hexadecimal_number(to_, val)

print(result, end=' ')

if to_ == 2:
    print("(2)")
elif to_ == 8:
    print("(8)")
elif to_ == 10:
    print("(10)")
else:
    print("(16)")

# 입력 예시
# 1)
# 2 16
# 10001
# 2)
# 16 8
# 11

# 출력 예시
# 1)
# 11 (16)
# 2)
# 21 (8)


import random

def potencia (a, b, p):

	if not b >= 1 or not p >= 2:
		return 'ERROR!! Parametros incorrectos (uso: b >= 1 y p >= 2)'

	x = 1

	while b != 1:
		if b % 2 == 1:
			x = (x * a) % p

		a = (a ** 2) % p
		b = b // 2

	x = (x * a) % p

	return x  # Devuelvo a^b (mod p)

def alpha_x_y(P,Q,c):

	#Calculo de Alpha:
	#Basado en http://es.wikipedia.org/wiki/DSA
	h=		random.randint(2,P-2)
	alpha=	potencia(h,c,P)
	x=		random.randint(2,Q-2)
	y=		potencia(alpha,x,P)

	return alpha, y, x


from flask import Flask
from flask import request
import subprocess
import logging
formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s", "%d-%m-%y %H:%M:%S")

#convert approved ids to list
file = open("approvedIDs.txt", "r")
ids = file.read().split(".")

app = Flask(__name__)

def start():
    subprocess.call([r'D:/Minecraft servers/1.19/test/Run me.bat'])

@app.route("/minecraft/start", methods=["GET"])
def checkID():
    uid = request.args.get("uid")
    logging.info(f"UID: {uid}")

    if uid in ids:
        start()
        return "Success"

    else:
        return "Fail"


# -*- coding: utf-8 -*-
# author: Moli
# last update: 20161108

#python TF_process_animal_v1.py ../hmmerdb/animal/animaltfdb.hmm
# /mnt/ilustre/users/sanger-dev/app/database/refGenome/Animal/Primates/human/Homo_sapiens.GRCh38.pep.all.fa
# family_vs_DBD_animal_2.0.txt
# /mnt/ilustre/users/sanger-dev/workspace/20161116/Single_express_sample_v8_change_sample_name
# /Express/output/diff/featurecounts_count.txt.A_vs_B.edgeR.DE_results_name

#python TF_process_animal_v2.py family_vs_DBD_animal_2.0.txt /mnt/ilustre/users/sanger-dev/workspace/20161116/Single_express_sample_v8_change_sample_name/Express/output/diff/featurecounts_count.txt.A_vs_B.edgeR.DE_results_name

import re
import sys
import subprocess
#
'''
利用hmmer软件将未知序列比对到蛋白域中去
hmmscan <hmmdb> (seq)
'''
hmmer_path = '/mnt/ilustre/users/sanger-dev/app/bioinfo/align/hmmer-3.1b2-linux-intel-x86_64/binaries/'
# ref = '/mnt/ilustre/users/sanger-dev/sg-users/zhoumoli/TF/hmmerdb/planttfdb.hmm'
# query = '/mnt/ilustre/users/sanger-dev/sg-users/zhoumoli/TF/code/test_plant.fas'

cmd = "{}hmmscan {} {}".format(hmmer_path, sys.argv[1], sys.argv[2])

try:
    # subprocess.check_output(cmd, shell=True)
    res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    mi = res.communicate()
    out = list(mi)
    out = out[0].split('\n')
except subprocess.CalledProcessError:
    raise Exception("运行出错！")
#
family_db = sys.argv[3]
# family_db = 'O:/Users/moli.zhou/Desktop/family_vs_DBD_animal_2.0.txt'
query = [] #输入的蛋白序列号
phmmer = [] #hmmscan比对结果
domain = []
tfid = []
gene = [] #蛋白对应的gene序列号
i = -1

'''
读出具有一一对应关系的DBD与family名称
'''
family_id = []
DBD = []
with open(family_db,'r') as db:
    for line in db:
        line = line.strip()
        line = line.split("\t")
        family_id.append(line[0])
        DBD.append(line[1])

'''
逐行分析比对结果
提取输入序列的名称
& 对应的比对内容
'''
# file = 'Homo_sapiens_hmm.out'
# with open(file, 'r') as f:
for line in out:
    line = line.strip()
    tem = re.match('#.*', line)
    if tem:
        pass
    elif line == '':
        pass
    elif 'Query:' in line:
        x = line.split(' ')
        query.append(x[7])
        i += 1
        phmmer.append("")
    elif 'Description:' in line:
        line = line.strip()
        g = re.search('gene:([a-zA-Z0-9\.]*)\s', line)
        gene.append(g.group(1))
        i += 1
        phmmer.append("")
    else:
        phmmer[i] += line+"\n"

family =[]
for i in range(len(phmmer)):
    if ">>" in phmmer[i]:
        y = re.search(">>\s(.*)\s", phmmer[i])
        z = y.group(1).split()
        for j in range(len(DBD)):
            if z[0] == DBD[j]:
                family.append(family_id[j])
        if z[0] not in DBD:
            family.append("Other")
    else:
        family.append("None")

diff_gene_id = sys.argv[4]
# diff_gene_id = 'featurecounts_count.txt.A_vs_B.edgeR.DE_results_name'
diff = []
with open(diff_gene_id, 'r') as d:
    for line in d:
        line = line.strip()
        diff.append(line)

for k in range(len(query)):
    if family[k] == "None":
        pass
    else:
        if gene[k] in diff:
            # print query[k] +'\t'+ family[k] +'\t'+gene[k] +'\n'
            f_out = open("TF_result.txt","a+")
            f_out.write(query[k] +'\t'+ family[k] +'\t'+gene[k] +'\n')
            f_out.close()
        else:
            # print query[k] +'\t'+ family[k] +'\t'+'no' +'\n'
            f_out = open("TF_result.txt","a+")
            f_out.write(query[k] +'\t'+ family[k] +'\t'+'no' +'\n')
            f_out.close()