import sqlite3
from script import Goals, teams, Games
import time

#Separates the output of script.py
print("\nOutput from script.py.")
print("\n" * 10)
print("Output now is from Query.py.\n\n")

def SortByGroup():
    Test = True
    while Test:
        search = input("Grupo (A/B/C/D/E/F/G/H): ").upper()
        if search in ("A", "B", "C", "D", "E", "F", "G", "H"):
            search = "Group " + search
            Test = False
        else:
            print("Enter a valid letter!\n")
    #Print games by user group input
    for row in cursor.execute("SELECT * FROM FifaWorldCup WHERE MatchGroup = ?", (search,)):
        print('{0}: {1} {2} {3} on {4} in {5}'.format(row[1], row[2], row[3], row[4], row[5], row[6]))

def SortByAllGroups():
    #Print all  rows sorted by group
    for row in cursor.execute('SELECT * FROM FifaWorldCup ORDER BY MatchGroup'):
        print('{0} {1} {2}, {3} in {4}'.format(row[1], row[2], row[3], row[4], row[5]))

def SortByGamesTeam(Teams):
    Test = True
    print("\n\n")
    #Print games by user group input
    while Test:
        print("List of teams: ")
        print(Teams)
        search = input("Name of team (ex: Portugal, be sure the name has Capitalize letters "
                        + "and must be in english): ")
        if search in Teams:
            print("\n\n")
            for row in cursor.execute("SELECT * FROM FifaWorldCup WHERE TeamHome = ?", (search,)):
                print('{0}: {1} {2} {3} on {4} in {5}'.format(row[1], row[2], row[3], row[4], row[5], row[6]))
            for row in cursor.execute("SELECT * FROM FifaWorldCup WHERE TeamAway = ?", (search,)):
                print('{0}: {1} {2} {3} on {4} in {5}'.format(row[1], row[2], row[3], row[4], row[5], row[6]))
            Test = False
        else:
            print("\n" * 5 + "Enter a valid team name!")

def SortByLocation(Locals):
    Test = True
    while Test:
        print("All locations: ")
        print(Locals)
        search = input("\nLocation name (Please Copy & Paste from the list above without "
                        + "\'\' to search the name of location): ")
        if search in Locals:
            #Code that is executed when is valid input
            print("\n\n")
            for row in cursor.execute("SELECT * FROM FifaWorldCup WHERE Location = ?", (search,)):
                print('{0}: {1} {2} {3} on {4} - {5}'.format(row[6], row[2], row[3], row[4], row[5], row[1]))
            Test = False
        else:
            print("\n" * 5 + "Enter a valid location!")

def SortByAllLocation():
    for row in cursor.execute("SELECT * FROM FifaWorldCup ORDER BY Location"):
        print('{0}: {1} {2} {3} on {4} - {5}'.format(row[6], row[2], row[3], row[4], row[5], row[1]))

def GoalsByTeam(Teams, Goals):
    Test = True
    golos = 0
    print("\n\n")
    #Print games by user group input
    while Test:
        print("List of teams: ")
        print(Teams)
        search = input("Name of team (ex: Portugal, be sure the name has Capitalize letters "
                        + "and must be in english): ")
        if search in Teams:
            #Code that is executed when is valid input
            j = [i for i, x in enumerate(teams) if x == search]
            try:
                for i in range(len(j)):
                    golos += Goals[j[i]]
            except IndexError:
                print(search + ": {0}".format(golos) + " goals")
            Test = False
        else:
            print("\n" * 5 + "Enter a valid team name!")

def AvGoalsPerGame(Golos, Games):
    av = Golos / Games
    print("Average goals per game:", av)

def AvGoalsPerDay(Golos):
    av = Golos / 14
    print("Average goals per day:", av)

def GoalsPerGroup(Goals):
    golosG = 0
    Test = True
    for row in cursor.execute("SELECT MatchGroup, GH, GA FROM FifaWorldCup ORDER BY MatchGroup"):
        if (row[1] and row [2]) > (-1):
            golosG += row[1] + row[2]
            Test = True
        elif Test:
            print('{0}: {1}'.format(row[0], golosG))
            golosG = 0
            Test = False

def EveryGoalRegistered(Golos):
    print("Total number of goals so far:", Golos)

#Connects to db
db = sqlite3.connect('db.sqlite3')
cursor = db.cursor()

Golos = 0
for i in range(len(Goals)):
    Golos += Goals[i]
Teams = []
for row in cursor.execute('''SELECT DISTINCT TeamHome FROM FifaWorldCup ORDER BY TeamHome'''):
    Teams.append(row[0])
Locals = []
for row in cursor.execute('''SELECT DISTINCT Location FROM FifaWorldCup ORDER BY Location'''):
    Locals.append(row[0])

Loop = True
while Loop:
    time.sleep(2)
    print("\n")
    print("1 - User Group Info | 2 - All groups by order | 3 - User Team Info | 4 - User Location Info "
            + "| 5 - All locations by order | 6 - User team goals | 7 - Average goals per game "
            + "| 8 - Average goals per day | 9 - User Group Goals | 10 - Every goal so far | 0 - Exit")
    op = input("Opção: ")
    if op == "0":
        exit()
    elif op == "1":
        print("\n" * 5)
        SortByGroup()
    elif op == "2":
        print("\n" * 5)
        SortByAllGroups()
    elif op == "3":
        print("\n" * 5)
        SortByGamesTeam(Teams)
    elif op == "4":
        print("\n" * 5)
        SortByLocation(Locals)
    elif op == "5":
        print("\n" * 5)
        SortByAllLocation()
    elif op == "6":
        print("\n" * 5)
        GoalsByTeam(Teams, Goals)
    elif op == "7":
        print("\n" * 5)
        AvGoalsPerGame(Golos, Games)
    elif op == "8":
        print("\n" * 5)
        AvGoalsPerDay(Golos)
    elif op == "9":
        print("\n" * 5)
        GoalsPerGroup(Goals)
    elif op == "10":
        print("\n" * 5)
        EveryGoalRegistered(Golos)

db.close()


"""Mixins for review request views."""

from __future__ import annotations

import logging
from datetime import datetime
from typing import List, Optional, TYPE_CHECKING

from django.db.models import Q
from django.http import Http404, HttpRequest, HttpResponse
from django.shortcuts import get_object_or_404, render
from django.template.defaultfilters import date
from django.utils.formats import localize
from django.utils.html import format_html
from django.utils.safestring import SafeString, mark_safe
from django.utils.timezone import localtime
from django.utils.translation import gettext
from djblets.views.generic.base import (CheckRequestMethodViewMixin,
                                        PrePostDispatchViewMixin)
from typing_extensions import TypedDict

from reviewboard.accounts.mixins import CheckLoginRequiredViewMixin
from reviewboard.diffviewer.models import DiffSet
from reviewboard.reviews.models import ReviewRequest, ReviewRequestDraft
from reviewboard.reviews.models.base_review_request_details import \
    BaseReviewRequestDetails
from reviewboard.reviews.models.review_request import ReviewRequestCloseInfo
from reviewboard.site.mixins import CheckLocalSiteAccessViewMixin
from reviewboard.site.models import LocalSite

if TYPE_CHECKING:
    from reviewboard.attachments.models import FileAttachmentSequence

logger = logging.getLogger(__name__)

class ReviewRequestViewMixin(CheckRequestMethodViewMixin,
                             CheckLoginRequiredViewMixin,
                             CheckLocalSiteAccessViewMixin,
                             PrePostDispatchViewMixin):
    """Common functionality for all review request-related pages.

    This performs checks to ensure that the user has access to the page,
    returning an error page if not. It also provides common functionality
    for fetching a review request for the given page, returning suitable
    context for the template, and generating an image used to represent
    the site when posting to social media sites.
    """

    permission_denied_template_name = \
        'reviews/review_request_permission_denied.html'

    class StatusExtraInfo(TypedDict):
        """Extra info to include in the status rendering.

        This is used in :py:meth:`get_review_request_status_html`.
        """

        #: A text string to display to the user.
        #:
        #: Type:
        #:     str
        text: str

        #: The timestamp associated with this info block.
        #:
        #: Type:
        #:     datetime.datetime
        timestamp: Optional[datetime]

        #: Extra info to format into the ``text`` string.
        #:
        #: Type:
        #:     dict
        extra_vars: dict

    def pre_dispatch(
        self,
        request: HttpRequest,
        review_request_id: int,
        *args,
        **kwargs,
    ) -> Optional[HttpResponse]:
        """Look up objects and permissions before dispatching the request.

        This will first look up the review request, returning an error page
        if it's not accessible. It will then store the review request before
        calling the handler for the HTTP request.

        Args:
            request (django.http.HttpRequest):
                The HTTP request from the client.

            review_request_id (int):
                The ID of the review request being accessed.

            *args (tuple):
                Positional arguments to pass to the handler.

            **kwargs (dict):
                Keyword arguments to pass to the handler.

                These will be arguments provided by the URL pattern.

        Returns:
            django.http.HttpResponse:
            The resulting HTTP response to send to the client, if there's
            a Permission Denied.
        """
        self.review_request = self.get_review_request(
            review_request_id=review_request_id,
            local_site=self.local_site)

        if not self.review_request.is_accessible_by(request.user):
            return self.render_permission_denied(request)

        return None

    def render_permission_denied(
        self,
        request: HttpRequest,
    ) -> HttpResponse:
        """Render a Permission Denied page.

        This will be shown to the user if they're not able to view the
        review request.

        Args:
            request (django.http.HttpRequest):
                The HTTP request from the client.

        Returns:
            django.http.HttpResponse:
            The resulting HTTP response to send to the client.
        """
        return render(request,
                      self.permission_denied_template_name,
                      status=403)

    def get_review_request(
        self,
        review_request_id: int,
        local_site: Optional[LocalSite] = None,
    ) -> ReviewRequest:
        """Return the review request for the given display ID.

        Args:
            review_request_id (int):
                The review request's display ID.

            local_site (reviewboard.site.models.LocalSite):
                The Local Site the review request is on.

        Returns:
            reviewboard.reviews.models.review_request.ReviewRequest:
            The review request for the given display ID and Local Site.

        Raises:
            django.http.Http404:
                The review request could not be found.
        """
        q = ReviewRequest.objects.all()

        if local_site:
            q = q.filter(local_site=local_site,
                         local_id=review_request_id)
        else:
            q = q.filter(pk=review_request_id)

        q = q.select_related('submitter', 'repository')

        return get_object_or_404(q)

    def get_diff(
        self,
        revision: Optional[int] = None,
        draft: Optional[ReviewRequestDraft] = None,
    ) -> DiffSet:
        """Return a diff on the review request matching the given criteria.

        If a draft is provided, and ``revision`` is either ``None`` or matches
        the revision on the draft's DiffSet, that DiffSet will be returned.

        Args:
            revision (int, optional):
                The revision of the diff to retrieve. If not provided, the
                latest DiffSet will be returned.

            draft (reviewboard.reviews.models.review_request_draft.
                   ReviewRequestDraft, optional):
                The draft of the review request.

        Returns:
            reviewboard.diffviewer.models.diffset.DiffSet:
            The resulting DiffSet.

        Raises:
            django.http.Http404:
                The diff does not exist.
        """
        # Normalize the revision, since it might come in as a string.
        if revision:
            revision = int(revision)

        # This will try to grab the diff associated with a draft if the review
        # request has an associated draft and is either the revision being
        # requested or no revision is being requested.
        if (draft and draft.diffset_id and
            (revision is None or draft.diffset.revision == revision)):
            return draft.diffset

        query = Q(history=self.review_request.diffset_history_id)

        # Grab a revision if requested.
        if revision is not None:
            query = query & Q(revision=revision)

        try:
            return DiffSet.objects.filter(query).latest()
        except DiffSet.DoesNotExist:
            raise Http404

    def get_social_page_image_url(
        self,
        file_attachments: FileAttachmentSequence,
    ) -> Optional[str]:
        """Return the URL to an image used for social media sharing.

        This will look for the first attachment in a list of attachments that
        can be used to represent the review request on social media sites and
        chat services. If a suitable attachment is found, its URL will be
        returned.

        Args:
            file_attachments (list of reviewboard.attachments.models.
                              FileAttachment):
                A list of file attachments used on a review request.

        Returns:
            str:
            The URL to the first image file attachment, if found, or ``None``
            if no suitable attachments were found.
        """
        for file_attachment in file_attachments:
            if file_attachment.mimetype.startswith('image/'):
                return file_attachment.get_absolute_url()

        return None

    def get_review_request_status_html(
        self,
        review_request_details: BaseReviewRequestDetails,
        close_info: ReviewRequestCloseInfo,
        extra_info: List[ReviewRequestViewMixin.StatusExtraInfo] = [],
    ) -> SafeString:
        """Return HTML describing the current status of a review request.

        This will return a description of the submitted, discarded, or open
        state for the review request, for use in the rendering of the page.

        Args:
            review_request_details (reviewboard.reviews.models
                                    .base_review_request_details
                                    .BaseReviewRequestDetails):
                The review request or draft being viewed.

            close_info (dict):
                A dictionary of information on the closed state of the
                review request.

            extra_info (list of dict):
                A list of dictionaries showing additional status information.
                Each must have a ``text`` field containing a format string
                using ``{keyword}``-formatted variables, a ``timestamp`` field
                (which will be normalized to the local timestamp), and an
                optional ``extra_vars`` for the format string.

        Returns:
            django.utils.safestring.SafeString:
            The status text as HTML for the page.
        """
        review_request = self.review_request
        status = review_request.status
        review_request_details = review_request_details

        if status == ReviewRequest.SUBMITTED:
            timestamp = close_info['timestamp']

            if timestamp:
                text = gettext('Created {created_time} and submitted '
                               '{timestamp}')
            else:
                text = gettext('Created {created_time} and submitted')
        elif status == ReviewRequest.DISCARDED:
            timestamp = close_info['timestamp']

            if timestamp:
                text = gettext('Created {created_time} and discarded '
                               '{timestamp}')
            else:
                text = gettext('Created {created_time} and discarded')
        elif status == ReviewRequest.PENDING_REVIEW:
            text = gettext('Created {created_time} and updated {timestamp}')
            timestamp = review_request_details.last_updated
        else:
            logger.error('Unexpected review request status %r for '
                         'review request %s',
                         status, review_request.display_id,
                         extra={'request': request})

            return mark_safe('')

        parts = [
            {
                'text': text,
                'timestamp': timestamp,
                'extra_vars': {
                    'created_time': date(localtime(review_request.time_added)),
                },
            },
        ] + extra_info

        html_parts = []

        for part in parts:
            if part['timestamp']:
                timestamp = localtime(part['timestamp'])
                timestamp_html = format_html(
                    '<time class="timesince" datetime="{0}">{1}</time>',
                    timestamp.isoformat(),
                    localize(timestamp))
            else:
                timestamp_html = ''

            html_parts.append(format_html(
                part['text'],
                timestamp=timestamp_html,
                **part.get('extra_vars', {})))

        return mark_safe(' — '.join(html_parts))


import cgi
from google.appengine.api import users

import app.model
import app.web

class User (app.web.UserRequestHandler):

    def get (self, user_id):
        self.set_module(__name__ + ".__init__")
        self.db_user = self.get_user(user_id)
        self.content["user_name"] = user_id

        if self.db_user:
            self.get_200()
        else:
            self.get_404()

    def get_200 (self):
        self.write_out("./200.html")

    def get_404 (self):
        self.error(404)
        self.write_out("./404.html")


# -*- coding: utf-8 -*-
"""
Created on Thu Oct 25 20:47:43 2018

@author: ACER
"""

#!/usr/bin/env python3

from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
import sys

class Window(QWidget):
    def __init__(self):
        QWidget.__init__(self)

        layout = QGridLayout()
        self.setLayout(layout)

        label = QLabel("The Story of Dale")
        layout.addWidget(label, 0,0)

        label = QLabel("Few people could understand Dale's motivation. It wasn't something that was fine")
        label.setWordWrap(True)
        layout.addWidget(label, 0,1)

app = QApplication(sys.argv)

screen = Window()
screen.show()
sys.exit(app.exec_())


from django.contrib.auth import login
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType

from rest_framework import permissions
from rest_framework.authtoken.serializers import AuthTokenSerializer
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from rest_framework.renderers import JSONRenderer

from knox.views import LoginView as KnoxLoginView

from .models import UserFileManager
from .serializers import UserFileManagerSerializer, PermissionSerializer
from .permissions import IsOwnerOrReadOnly

class LoginView(KnoxLoginView):
    permission_classes = (permissions.AllowAny,)
    authentication_classes = []

    def post(self, request, format=None):

        serializer = AuthTokenSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.validated_data['user']
        login(request, user)
        return super(LoginView, self).post(request, format=None)

class UserFileManagerListView(APIView):
    """
    List all users or create a new user
    """

    permission_classes = [permissions.IsAuthenticated, IsOwnerOrReadOnly]

    def get(self, request):
        users = UserFileManager.objects.all()
        serializer = UserFileManagerSerializer(users, many=True)
        return Response({'data': serializer.data})

    def post(self, request):
        serializer = UserFileManagerSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(
                {'data': serializer.data},
                status=status.HTTP_201_CREATED
            )
        return Response(
            {'error': serializer.errors},
            status=status.HTTP_400_BAD_REQUEST
        )

class PermissionListView(APIView):
    """
    List all permission or create a new permission.

    The new permission is created only in relation of `UserFileManager` model
    as a `content_type`
    """

    def get(self, request):
        django_permissions = Permission.objects.all()
        serializer = PermissionSerializer(django_permissions, many=True)
        return Response({'data': serializer.data})

    def post(self, request):
        codename = request.data['codename']
        name = request.data['name']
        content_type = ContentType.objects.get_for_model(UserFileManager)

        data = {
            'codename': codename,
            'name': name,
            'content_type': content_type.pk
        }

        serializer = PermissionSerializer(data=data)

        if serializer.is_valid():
            serializer.save()
            return Response(
                {'message': 'A new permission is created', 'created_permission': serializer.data},
                status.HTTP_201_CREATED
            )

        return Response(
            {'error': serializer.errors},
            status=status.HTTP_400_BAD_REQUEST
        )


import urllib

import os
from time import sleep

import pytz
import sys
from PIL import Image, ImageFilter
from bs4 import BeautifulSoup
from initial_setup import post_installclass
from khayyam import JalaliDatetime, TehranTimezone

from core.function.image_handler import watermarkimage
from core.function.log_handler import LogHandler
from dal.news_repository import NewsRepository

class BeytooteScrapper:
    posts_to_add = []
    posts_to_add_urls = []
    popular_news_url = "http://www.beytoote.com/hotnews.html"
    custom_hashtags = ["اخبار"]

    def __init__(self):
        # self.url = "http://www.beytoote.com/news/sporty-news/fanews3390.html"
        self.log_handler = LogHandler(log_mod=0, log_file="", log_file_path="")
        self.root_dir = os.path.dirname(__file__)
        self.news_repository = NewsRepository()

    def read_single_post(self, post_url):

        post = {}

        post["id"] = post_url[post_url.rfind("/") + 1:post_url.rfind(".")]

        fp = urllib.request.urlopen(post_url)
        my_bytes = fp.read()

        page = my_bytes.decode("utf8")
        fp.close()
        soup = BeautifulSoup(page, 'html.parser')
        post["title"] = soup.select("#content h2 a")[0]
        abstract = soup.select("#content .box-note")[0]

        source_date = soup.select("#content .published")[0].getText().replace('تاریخ انتشار :  ', "") \
            .replace('\n', "") \
            .replace('\t', "")
        shamsi_date = JalaliDatetime.strptime(source_date, '%A, %d %B %Y %H:%M')
        shamsi_datexx = JalaliDatetime(shamsi_date.year, shamsi_date.month, shamsi_date.day, shamsi_date.hour,
                                       shamsi_date.minute, shamsi_date.second, 0, TehranTimezone())

        source_date_utc = shamsi_datexx.todatetime().astimezone(pytz.timezone('UTC'))
        # localtime_shamsi_date = local.localize(source_date_local, is_dst=None)

        image_src = soup.select("#content p.imgarticle img")[0]["src"]
        post["source_media_url"] = "http://beytoote.com/" + image_src
        main_image_file_name = post["source_media_url"][post["source_media_url"].rfind("/") + 1:]
        post["media_type"] = 2

        all_paragraphs = soup.select("#content p")
        post["body"] = ""
        for p in all_paragraphs:
            paragraph_text = p.getText().strip()
            if len(paragraph_text) > 0:
                post["body"] = post["body"] + paragraph_text + "\n"
                # print(paragraph_text)

        post["date"] = source_date_utc
        return post
        # print(source_date)
        # now = JalaliDatetime.strptime("جمعه, 16 تیر 1396 22:26", '%A, %d %b %Y %H:%M')
        # print(source_date_utc)

    def insert_to_db(self):
        for post_url in self.posts_to_add_urls:
            post = self.read_single_post(post_url)
            is_video = False
            self.log_handler.write_log("insert_to_db tweet id : %s" % post['id'])
            directory = os.path.join(self.root_dir, os.path.realpath('cdn/content/news/btt_' + post['id']))
            if not os.path.exists(directory):
                os.makedirs(directory)

            file_name_thumbnail = 'thumbnail_' + post['id'] + ".jpg"
            file_path_thumbnail = os.path.join(directory, file_name_thumbnail)

            file_name = None
            post["media_type"] = 1
            has_media = True
            if has_media:
                if not os.path.exists(file_path_thumbnail):
                    urllib.request.urlretrieve(post["source_media_url"], file_path_thumbnail)

                thumb_image = Image.open(file_path_thumbnail)
                thumb_image_width = 400
                thumb_image_height = int((thumb_image_width / thumb_image.width) * thumb_image.height)
                thumb_size = thumb_image_width, thumb_image_height
                if is_video:  # hasvideo
                    thumb_image = thumb_image.resize(thumb_size).filter(ImageFilter.GaussianBlur(radius=10))
                    thumb_image.save(file_path_thumbnail)
                    play_image_path = os.path.join(self.root_dir, os.path.realpath('cdn/image/play.png'))
                    watermarkimage(file_path_thumbnail, play_image_path)
                else:
                    thumb_image = thumb_image.resize(thumb_size)
                    thumb_image.save(file_path_thumbnail)

                thumbnail = {}
                thumbnail["image_url"] = "/content/news/btt_" + post['id'] + "/" + file_name_thumbnail
                thumbnail["image_width"], thumbnail["image_height"] = thumb_image.size
                thumbnail["image_aspect_ratio"] = thumbnail["image_width"] / thumbnail["image_height"]

                file_extention = ".jpg"

                post["media_type"] = 2  ##### Image
                if is_video:
                    file_extention = ".mp4"
                    # download_url = tweet["extended_entities"]["media"][0]["video_info"]["variants"][1]["url"]
                    # tweet["media_type"] = 3  ##### Video

                file_name = post['id'] + file_extention
                file_path = os.path.realpath(directory + '/' + file_name)

                if not os.path.exists(file_path):
                    urllib.request.urlretrieve(post["source_media_url"], file_path)

                main_image = Image.open(file_path)

                post["media_width"] = main_image.width
                post["media_height"] = main_image.height
                post["media_aspect_ratio"] = post["media_width"] / post["media_height"]

                post["thumbnail"] = thumbnail

                post["source_owner_id"] = "beytoote"
                post["source_owner_alias"] = "beytoote"
                post["source_owner_title"] = "بیطوطه"

            try:
                self.news_repository.upsert_news_post(post, file_name, self.custom_hashtags)
                sleep(1)
            except:
                self.log_handler.write_log("Error: upsert_twitter_post : %s" % post['id_str'])
                raise

    def get_popular_news(self):
        try:
            fp = urllib.request.urlopen(self.popular_news_url)
            my_bytes = fp.read()

            page = my_bytes.decode("utf8")
            fp.close()
            news_to_add = []
            soup = BeautifulSoup(page, 'html.parser')
            links = soup.select(".hot-news .allmode_topitem .allmode_title a")

            self.log_handler.write_log("%s Tweets to insert..." % len(news_to_add))

            for link in links:
                self.posts_to_add_urls.append("http://beytoote.com" + link["href"])

                # print(tweet_to_add )
        except:
            self.log_handler.write_log("Error: get_home_tweets=> %s" % sys.exc_info()[0])

    def auto_mod(self):
        # while True:
        if len(self.posts_to_add) == 0:  # or len(self.users_to_ollow) == 0:
            self.get_popular_news()
            self.insert_to_db()
            self.posts_to_add = []
            self.posts_to_add_urls = []

        # Bot iteration in 3600 sec
        sleep_time = 500
        self.log_handler.write_log("Sleeping %s seconds..." % sleep_time)
        sleep(sleep_time)

        # -----------------------------------------------------------------------
        # loop through each of my statuses, and print its content
        # -----------------------------------------------------------------------
        # for status in statuses:
        #     print("(%s) @%s %s" % (status["created_at"], status["user"]["screen_name"], status["text"]))

if __name__ == '__main__':
    ff = BeytooteScrapper()
    ff.auto_mod()


#! python3

''' The program will prompt for a URL,
read the JSON data from that URL using urllib
and then parse and extract the comment counts from the JSON data,
compute the sum of the numbers in the file

Sample data: http://python-data.dr-chuck.net/comments_42.json (Sum=2553)

Assignment #2, using JSON

Developed by Kyle DuBois, Version 1.0 '''

import urllib.request, urllib.parse, urllib.error
import json

numList = list()

while True:
    address = input('Enter location: ')
    if len(address) < 1:
        break

    print('Retrieving', address)

    # This section opens the URL and parses through JSON data
    url = urllib.request.urlopen(address)
    data = url.read().decode()
    headers = dict(url.getheaders())
    print('Retrieved', len(data),'characters')
    info = json.loads(data)

    #print(json.dumps(info, indent=4))

    # This section looks for and extracts the count numbers from the JSON data and creates a list
    for u in info["comments"]:
        #print(u['count'])
        numList.append(u['count'])

    # This section runs through the list and creates a new list of integers
    nunuList = list()
    for num in numList:
        newNum = int(num)
        nunuList.append(newNum)

    # This section sums all the integers and prints the total
    tot = sum(nunuList)
    print(tot)
    break


import numpy as np

class KMeans(object):
    def __init__(self, n_clusters=2, n_iter=300, shuffle=True, tolerance=0.001):
        self.n_iter = n_iter
        self.n_clusters = n_clusters
        self.tolerance = tolerance
        self.shuffle = shuffle
        self.run = False

    def fit(self, x):
        self.run = True
        self.centroids = {}

        if len(x.shape) < 1:
            raise Exception("DataException: Dataset must contain more examples" +
                            "than the required number of clusters!")
        if self.shuffle:
            r = np.random.permutation(x.shape[0])
            for k in range(len(r[:self.n_clusters])):
                self.centroids[k] = x[r[k]]
        else:
            for k in range(self.n_clusters):
                self.centroids[k] = x[k]

        for itr in range(self.n_iter):
            self.clusters = {}

            for k in range(self.n_clusters):
                self.clusters[k] = []

            for xi in x:
                dist = [np.linalg.norm(xi - self.centroids[c]) for c in self.centroids]
                class_ = dist.index(min(dist))
                self.clusters[class_].append(xi)

            old_centroids = dict(self.centroids)
            for k in self.clusters:
                self.centroids[k] = np.average(self.clusters[k], axis=0)

            is_done = True
            for k in self.centroids:
                old_centroid = old_centroids[k]
                centroid = self.centroids[k]
                if (np.linalg.norm(old_centroid - centroid) > self.tolerance):
                    is_done = False

            if is_done:
                break

    def predict(self, x):
        if self.run:
            if len(x.shape) > 1:
                class_ = []
                for c in self.centroids:
                    class_.append(np.sum((x - self.centroids[c]) ** 2, axis=1))
                return np.argmin(np.array(class_).T, axis=1)
            else:
                dist = [np.linalg.norm(x - self.centroids[c]) for c in self.centroids]
                class_ = dist.index(min(dist))
                return class_
        else:
            raise Exception("NonTrainedModelException: You must fit data first!")


'''
Collection of functions to drawing synthetic corpora based on Dirichlet distribution.
'''

# <<< Add the src path to sys.path
import sys
import os

containing_folder = os.path.dirname(os.path.abspath(__file__))
src_dir = os.path.abspath(os.path.join(containing_folder, os.pardir))
if src_dir not in sys.path:
    sys.path.insert(0, src_dir)
# >>>

import numpy as np

from corpora.synthetic import draw_dwz_from_ptd_pwt, get_pw_pt, draw_dirichlet
from corpora.structure import deltaI_from_nwd
from common.convert_states import state_nwjd, nwd_to_texts

def make_pwt_ptd_dirichlet(vec_alpha, vec_beta, D, seed=None):
    '''From Dirichlet hyperparameters, generate the conditional probabilities
    - p(t|d)
    - p(w|t)
    IN:
    - vec_alpha: hyperparameter for topic-doc distribution, len=K...number of topics
    - vec_beta: hyperparameter for word-topic distribution, len=V...number of word-types
    - D: number of documents, int
    OUT:
    - p_td:  p(t|d), shape=KxD
    - p_wt:  p(w|t), shape=VxK
    '''
    K = len(vec_alpha)
    V = len(vec_beta)

    # get seed_doc_list and seed_topic_list
    if seed is not None:
        np.random.seed(seed=seed)
        seed_doc_list = np.random.choice(D * D, size=D, replace=False)
        np.random.seed(seed=seed + 100)
        seed_topic_list = np.random.choice(K * K, size=K, replace=False)

    # # p_t_d from samples of alpha_vec
    p_td = np.zeros((K, D))
    for i_d in range(D):
        if seed is None:
            seed_doc = None
        else:
            seed_doc = seed_doc_list[i_d]
        p_td_tmp = draw_dirichlet(vec_alpha, seed=seed_doc)
        p_td[:, i_d] = 1.0 * p_td_tmp
    # # p_w_t from samples of beta_vec
    p_wt = np.zeros((V, K))
    for i_K in range(K):
        if seed is None:
            seed_topic = None
        else:
            seed_topic = seed_topic_list[i_K]
        p_wt_tmp = draw_dirichlet(vec_beta, seed=seed_topic)
        p_wt[:, i_K] = 1.0 * p_wt_tmp

    return p_td, p_wt

def make_hyper_vec(alpha, vec_p):
    '''make the hyperparameter vector in our notation.
    The resulting dirichlet-hyperparameter is given by:
    vec{alpha} = (alpha_1,alpha_2,...,alpha_S) with S = len(vec_p) with alpha_i = alpha*S*p_i.
    If we draw random variables vec{x}, the marginal expectation value is <x_i> = p_i

    IN:
    - alpha, float
    - vec_p = (p_i) with i=1,..,S
    OUT:
    - vec_alpha = (alpha_i) with i =1,...,S
    '''
    return alpha * len(vec_p) * vec_p

def synthetic_dirichlet_wrapper(dict_input, if_info=True):
    '''
    Wrapper for drawing synthetic corpus: DIRICHLET

    Input:
        dict_input = {
            ## choose topic model
            'corpustype': 'dirichlet', str

            ## corpus-size parameters
            , 'D':D, int  ;#of docs
              'V':V, int  ;#of word-types
              'm':m, int  ;textlength
              'K':K, int  ;#of topics

            ## structure parameters
              'alpha':alpha, float   ; concentration parameter for topic-doc distribution p_td
              'beta':beta, float   ; concentration parameter for word-topic distribution p_wt
            ## external contraints parameters
            'dist_w': dist_w, str ; global distribution of word-frequencies
                                    'uni','zipf'
            'dist_z': dist_z, str ; global distribution of topic-frequencies
                                    'uni','zipf'
        }

    Output:
        dict_output = {
            'state_dwz':[(d,w,z)]; list of 3-tuples for each drawn token (doc,word-type,topic-label)
            list_t_d_true:[ t_d]; topic-membership of each document
            'p_wt':p_wt; word-topic dist, arr.shape=VxK
            'p_td':p_td; topic-doc dist, arr.shape=KxD
        }
    '''

    # # get the size-parameters

    V = dict_input['V']
    K = dict_input['K']

    # # external contraints parameters
    dist_w = dict_input['dist_w']

    # # get the structure parameters
    alpha = dict_input['alpha']

    # # Parameters for document
    D = dict_input['D']
    m = dict_input['m']

    # # optional parameter
    beta = dict_input.get('beta', None)
    dist_t = dict_input.get('dist_t', None)
    seed = dict_input.get('seed', None)
    burstiness = dict_input.get('burstiness', None)

    #################################

    dict_output = synthetic_dirichlet_terminal(V, K, dist_w, D, m, alpha, beta, dist_t=dist_t, seed=seed, burstiness=burstiness, if_info=if_info)

    return dict_output

def synthetic_dirichlet_terminal(V, K, dist_w, D, m, alpha, beta=None, dist_t=None, seed=None, burstiness=None, if_info=True):

    if dist_t is None:
        dist_t = dist_w
    if beta is None:
        beta = 1.0 * alpha

    # # global distribution of topic-size
    p_t = get_pw_pt(K, dist=dist_t)
    # # global distribution of word frequencies
    p_w = get_pw_pt(V, dist=dist_w)

    # # get vector-hyperparameters
    vec_alpha = make_hyper_vec(alpha, p_t)
    vec_beta = make_hyper_vec(beta, p_w)

    # # create the mixture-matrices p_wt (word-topic) and p_td (topic-doc)
    p_td, p_wt = make_pwt_ptd_dirichlet(vec_alpha, vec_beta, D, seed=seed)

    # # draw the dwz-state
    state_dwz, p_w_td = draw_dwz_from_ptd_pwt(p_td, p_wt, m, burstiness=burstiness)
    n_wd, n_wj, n_jd = state_nwjd(state_dwz, D, V, K)
    texts = nwd_to_texts(n_wd)
    # # infer the topic-membership of each doc:
    # # choose topic with largest contribution from p(t|d)
    list_t_d_true = np.argmax(p_td, axis=0)

    # # empirical p_t and p_w; otherwise p_tw is not normalized
    p_t_emp = 1.0 / D * np.sum(p_td, axis=1)
    p_w_emp = np.sum(p_wt * p_t_emp)

    # # infer the topic-membership of each word:
    # # choose topic with largest contribution from p(t|w) = p(w|t)*p(t)/p(w)
    # p_tw = (p_wt*(p_w[:,np.newaxis]/p_t)).T
    p_tw = (p_wt.T * (p_t_emp[:, np.newaxis] / p_w_emp))
    list_t_w_true = np.argmax(p_tw, axis=0)

    # # Get the structure
    if if_info:
        DeltaI, I_alpha = deltaI_from_nwd(n_wd)
    else:
        DeltaI = 0
        I_alpha = 0

    dict_out = {}
    dict_out['p_w'] = p_w
    # dict_out['V_t'] =V_t
    dict_out['word_topic_assign_list'] = list_t_w_true
    dict_out['p_t'] = p_t
    dict_out['p_wt'] = p_wt
    dict_out['document_topic_assign_list'] = list_t_d_true
    dict_out['p_td'] = p_td
    dict_out['state_dwz'] = state_dwz
    dict_out['n_wd'] = n_wd
    dict_out['n_wj'] = n_wj
    dict_out['n_jd'] = n_jd
    dict_out['texts'] = texts
    dict_out['p_tw'] = p_tw

    dict_out['DeltaI'] = DeltaI
    dict_out['I_alpha'] = I_alpha

    return dict_out


#!/usr/bin/env python
# coding: utf-8

# # Prepare

import json
import re
from convlab2.util.multiwoz.multiwoz_slot_trans import *
en_pattern = re.compile('[a-zA-Z]')

vocab_dict_path = 'vocab_dict.json'
vocab_dict = json.load(open(vocab_dict_path))

# # MultiWOZ ontology translate

def capitalize(s):
    return s[0].upper() + s[1:]

def intent_translate(intent):
    return vocab_dict['intent_set'][intent.lower()]

def domain_translate(domain):
    return vocab_dict['domain_set'][domain.lower()]

def slot_translate(domain, slot):
    if slot == 'taxi_phone':
        slot = 'phone'
    elif slot == 'taxi_types':
        slot = 'car'

    try:
        if slot.lower() in vocab_dict['slot_set']:
            return vocab_dict['slot_set'][slot.lower()]
        elif capitalize(domain) in REF_USR_DA and slot.lower() in REF_USR_DA[capitalize(domain)]:
            return vocab_dict['slot_set'][REF_USR_DA[capitalize(domain)][slot.lower()].lower()]
        elif capitalize(domain) in REF_SYS_DA and capitalize(slot) in REF_SYS_DA[capitalize(domain)]:
            return vocab_dict['slot_set'][REF_SYS_DA[capitalize(domain)][capitalize(slot)].lower()]

    except:
        print(domain, slot)
        raise

def value_translate(domain, slot, value):
    value = value.strip().lower()
    error_tup = []
    if (domain, slot, value) in error_tup:
        print(f'    Skip mismatching tuple: {domain}-{slot}-{value}')
        return ''

    if domain.lower() == 'general':
        return value

    if domain not in vocab_dict['value_set']:
        domain = domain.lower()

    if slot not in vocab_dict['value_set'][domain]:
        slot = slot.lower()

    if slot not in vocab_dict['value_set'][domain] and slot.lower() in REF_USR_DA[capitalize(domain)]:
        slot = REF_USR_DA[capitalize(domain)][slot.lower()]

    if slot not in vocab_dict['value_set'][domain] and capitalize(slot) in REF_SYS_DA[capitalize(domain)]:
        slot = REF_SYS_DA[capitalize(domain)][capitalize(slot)]

    if slot not in vocab_dict['value_set'][domain]:
        if slot is None:
            slot = 'none'
    if slot not in vocab_dict['value_set'][domain]:
        print(f'{domain}-{slot} has no vocab dict')
        raise Exception

    if not en_pattern.findall(value):
        return value

    if not vocab_dict['value_set'][domain][slot]:
        return value

    if slot in ['post', 'postcode', 'phone', 'Ref', 'reference']:
        return value

    if value not in vocab_dict['value_set'][domain][slot]:
        if '|' in value:
            return '或'.join([value_translate(domain, slot, v) for v in value.split('|')])

        trans_value = {
            "king's college": "king 's college",
            "intermediate dependency area": "intermediate dependancy area"
            # "saint john's college": "saint johns college",
            # "kettle's yard": "kettles yard",
            # "christ's college": "christ college",
            # "little saint mary's church": "little saint marys church",
            # "people's portraits exhibition at girton college": "peoples portraits exhibition at girton college",
        }
        if value in trans_value:
            value = trans_value[value]
        elif value.replace("'", "") in vocab_dict['value_set'][domain][slot]:
            value = value.replace("'", "")
        elif value.replace("'s", "") in vocab_dict['value_set'][domain][slot]:
            value = value.replace("'s", "")
        elif value.replace('the ', '') in vocab_dict['value_set'][domain][slot]:
            value = value.replace('the ', '')

    try:
        return vocab_dict['value_set'][domain][slot][value]
    except:
        print(domain, slot, value)
        raise Exception

def ontology_translate(typ, *args):
    assert typ in ['intent', 'domain', 'slot', 'value'], 'Function translate() requires 1st argument: domain|slot|value.'
    if typ == 'intent':
        assert len(args) == 1, 'Needs 1 argument: intent.'
        return intent_translate(args[0])
    elif typ == 'domain':
        assert len(args) == 1, 'Needs 1 argument: domain.'
        return domain_translate(args[0])
    elif typ == 'slot':
        assert len(args) == 2, 'Needs 2 argument: domain, slot.'
        return slot_translate(args[0], args[1])
    elif typ == 'value':
        assert len(args) == 3, 'Needs 3 argument: domain, slot, value.'
        return value_translate(args[0], args[1], args[2])

if __name__ == '__main__':
    print(ontology_translate('intent', 'request'))
    print(ontology_translate('domain', 'police'))
    print(ontology_translate('slot', 'attraction', 'phone'))
    print(ontology_translate('value', 'attraction', 'area', 'central district'))


import argparse
from my_dataset import MyDataSet as data
from model import UFUformer as net
import cv2
import os
import random
import numpy as np
from tensorboardX import SummaryWriter
from torch.optim.lr_scheduler import StepLR
import torch
import torch.nn as nn
import torchvision.utils as vutils
from torch.utils.data import DataLoader
from tqdm import tqdm

ap=argparse.ArgumentParser()
ap.add_argument('-b','--batch',help='the number of batch',type=int,default='4')
ap.add_argument('-e','--epoch',help='the number of training',type=int,default='500')
ap.add_argument('-r','--resume',help='the choice of resume',type=bool,default=False)

args=vars(ap.parse_args())

def log_images(writer, img, out,ll256,gt, it):

    images_array = vutils.make_grid(img).to('cpu')
    out_array = vutils.make_grid(out * 255).to('cpu').detach()
    ll256_array = vutils.make_grid(ll256 * 255).to('cpu').detach()
    gt = vutils.make_grid(gt).to('cpu')

    writer.add_image('input', images_array, it)
    writer.add_image('out', out_array, it)
    writer.add_image('ll256',ll256_array,it)
    writer.add_image('gt',gt,it)

net=net()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

if args['resume']:
    files = '.\\checkpoint\\checkpoint_255_epoch.pkl'
    assert os.path.isfile(files), "{} is not a file.".format(args['resume'])
    state = torch.load(files)
    net.load_state_dict(state['model'])
    iteration = state['epoch'] + 1
    optimizer = state['optimizer']
    print("Checkpoint is loaded at {} | epochs: {}".format(args['resume'], iteration))
else:
    iteration = 0

da=data('D:\\SZ\\UIEB')
dataloder=DataLoader(da,batch_size=args['batch'],shuffle=True)

optimizer=torch.optim.Adam(lr=0.0001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0,params=net.parameters())
scheduler = StepLR(optimizer, step_size=150, gamma=0.5)

def proimage(im):
    images = im[image_idx, :, :, :].clone().detach().requires_grad_(False)
    image = torch.transpose(images, 0, 1)
    image = torch.transpose(image, 1, 2).cpu().numpy() * 255
    return image

writer = SummaryWriter(comment='test_comment', filename_suffix="test_suffix")
j = 0
h = 0
for iter in range(iteration,args['epoch']):
    print(iter)
    prob = tqdm(enumerate(dataloder),total=len(dataloder))
    if iter < 1000:
        L1 = nn.MSELoss()
    else:
        L1 = nn.L1Loss()
    for i,data in prob:
        gt=torch.tensor(data[0].numpy(),requires_grad=True,device='cuda')
        raw=torch.tensor(data[1].numpy(),requires_grad=True,device='cuda')
        net.to('cuda')
        a=net(raw)
        L1loss = L1(a,gt)
        loss = L1loss
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        prob.set_postfix(Loss1=L1loss)
        h = h+1
        writer.add_scalar('loss',loss,h)

        c = a
        if i % 100 ==0:
            j += 100
            image_idx =random.randint(0, 0)
            predi=c[image_idx,:,:,:].clone().detach().requires_grad_(False)
            predi=torch.transpose(predi,0,1)
            predi=torch.transpose(predi,1,2).cpu().numpy()*255
            gti=proimage(gt)
            rawi=proimage(raw)
            image=np.concatenate((rawi,predi,gti),axis=1)
            image_name='sample12' + "/out" + str(iter)+'_'+str(i) + ".png"
            cv2.imwrite(image_name,image)

    if (iter + 1) % 1 == 0:
        checkpoint = {"model": net.state_dict(),
                      "optimizer": optimizer,
                      "epoch": iter}

        if not os.path.exists('checkpoint'):
            os.mkdir('checkpoint')
        path_checkpoint = "checkpoint/checkpoint_{}_epoch.pkl".format(iter)
        torch.save(checkpoint, path_checkpoint)

    scheduler.step()
    print(optimizer.param_groups[0]['lr'])


# Input data paths

data_dir = "./data"
covariates = f"{data_dir}/Covariates_simulation.csv"
thickness = f"{data_dir}/CorticalMeasuresENIGMA_ThickAvg.csv"
volume = f"{data_dir}/SubcorticalMeasuresENIGMA_VolAvg.csv"
output_dir = f"./output"

nj = -1

#-----------------------------------------------------------------------------------------------------------------

# Import dependencies

import pandas as pd
import numpy as np
from joblib import Parallel, delayed
import warnings
import networkx as nx
from scipy import stats
from sklearn.linear_model import LinearRegression
import logging
import argparse

logging.basicConfig(filename=f"{output_dir}/log.txt",
    format='%(asctime)s %(levelname)-8s %(message)s',
    level=logging.DEBUG,
    datefmt='%Y-%m-%d %H:%M:%S %Z')
existing = []
def log_func(existing, var_dict):
	new = {key:type(value) for key, value in var_dict.items() if key not in existing}
	existing.extend([key for key in var_dict if key not in existing])
	return new, existing
#-----------------------------------------------------------------------------------------------------------------

# Assemble dataframe for analyses and screen subjects

demographics = pd.read_csv(covariates, index_col="SubjID")
dashless_columns = pd.Series(demographics.columns.str.split('_')).apply(lambda x: ''.join(str.capitalize(s) for s in x))
demographics.rename(columns=dict(zip(demographics.columns, dashless_columns)), inplace=True)

demographics.loc[demographics.Subtype.isna(), 'Subtype'] = 0
demographics['Minor'] = np.int32(demographics.Age<18.)
demographics['Durill3'] = np.int32(demographics.Durill<3.)
demographics['Age2'] = demographics['Age']**2

thickness_volume = pd.read_csv(thickness, index_col="SubjID", usecols=lambda x: x!='ICV').merge(pd.read_csv(volume, index_col="SubjID"), left_index=True, right_index=True)
dashless_columns = pd.Series(thickness_volume.columns.str.split('_')).apply(lambda x: ''.join(str.capitalize(s) for s in x))
thickness_volume.rename(columns=dict(zip(thickness_volume.columns, dashless_columns)), inplace=True)

macroscale_indices = ['Lthickness', 'Rthickness', 'Lsurfarea', 'Rsurfarea', 'Icv']
demographics = demographics.merge(thickness_volume[macroscale_indices], left_index=True, right_index=True)
thickness_volume = thickness_volume.drop(macroscale_indices, axis=1)

covar_regressors = ['Age', 'Hand']
to_keep = np.all(~demographics[covar_regressors].isna(), axis=1)
demographics = demographics.loc[to_keep, :]
thickness_volume = thickness_volume.loc[to_keep, :]

demographics.to_csv(f"{data_dir}/Demographics.csv")
thickness_volume.to_csv(f"{data_dir}/CT_Volume.csv")

logging.debug(f"DATAFRAME CREATED\n{log_func(existing, locals())[0]}\n\n")
#-----------------------------------------------------------------------------------------------------------------

# Correct for covariates

def covar_correct(X, Y, data):

    def get_resid(X, y, data):
        r = data[y] - LinearRegression(n_jobs=1).fit(data[X], data[y]).predict(data[X])
        return r

    R = Parallel(n_jobs=nj)(delayed(get_resid)(X, y, data) for y in Y)
    return np.asanyarray(R).T

demographics = pd.read_csv(f"{data_dir}/Demographics.csv", index_col="SubjID")
thickness_volume = pd.read_csv(f"{data_dir}/CT_Volume.csv", index_col="SubjID")
brain_regions = thickness_volume.columns

data = demographics[['Age', 'Hand']].merge(thickness_volume, left_index=True, right_index=True)

X = ['Age', 'Hand']
Y = brain_regions

residuals = thickness_volume.copy()
residuals[brain_regions] = covar_correct(X, Y, data)

residuals.to_csv(f"{data_dir}/Data_residuals.csv")

logging.debug(f"DATA CORRECTED FOR CONFOUNDS\n{log_func(existing, locals())[0]}\n\n")

#-----------------------------------------------------------------------------------------------------------------

# Normalize on healthy subjects

thickness_volume = pd.read_csv(f"{data_dir}/CT_Volume.csv", index_col="SubjID")
demographics = pd.read_csv(f"{data_dir}/Demographics.csv", index_col="SubjID")
residuals = pd.read_csv(f"{data_dir}/Data_residuals.csv", index_col="SubjID")
residuals = thickness_volume.merge(demographics['Dx'], left_index=True, right_index=True)

mu = residuals.groupby('Dx').mean()
sd = residuals.groupby('Dx').std()

zscores = ((residuals - mu.loc[0]) / sd.loc[0]).drop('Dx', axis=1)

zscores.to_csv(f"{data_dir}/Data_zscores.csv")

logging.debug(f"DATA ZSCORED\n{log_func(existing, locals())[0]}\n\n")
#-----------------------------------------------------------------------------------------------------------------

# Generate graphs and extract metrics

thresholds = np.arange(95,101, 1)

demographics = pd.read_csv(f"{data_dir}/Demographics.csv", index_col="SubjID")
zscores = pd.read_csv(f"{data_dir}/Data_zscores.csv", index_col="SubjID")
demo_columns = demographics.columns
brain_regions = zscores.columns

def joint_variation(v):
    warnings.filterwarnings('ignore')
    if ~isinstance(v, np.ndarray):
        v = np.asanyarray(v)
    JV = np.array([ 1 / np.exp((v - i) ** 2) for i in v])
    np.fill_diagonal(JV, 0)
    return JV

def get_metrics(M, thr, nodes):
    M[M < np.percentile(M, thr)] = 0
    M[M > 0] = 1

    B = nx.from_numpy_matrix(M)
    B = nx.relabel_nodes(B, dict(zip(B, nodes)))
    B = nx.algorithms.full_diagnostics(B, swi=False, swi_niter=100, swi_nrand=10, swi_seed=None, n_jobs=nj, prefer=None)

    attributes = B.nodes[nodes[0]].keys()
    metric_dict = {metric: nx.get_node_attributes(B, metric) for metric in attributes}
    metric_dict.update({metric: {"global": value} for metric, value in B.metrics.items()})
    return metric_dict

results = {}
logging.debug(f"\n\nCOMPUTED METRICS FOR K:")
for thr in thresholds:
    r = Parallel(n_jobs=nj)(delayed(get_metrics)(joint_variation(v), thr, brain_regions) for _, v in zscores.iterrows())
    results.update({thr: dict(zip(zscores.index, r))})
    logging.debug(f"\t{thr}")

output_df = pd.DataFrame([[subj, thr] for subj in results[thr].keys() for thr in results.keys()], columns=['SubjID', 'Density']).set_index('SubjID')

metrics = results[thresholds[0]][zscores.index[0]].keys()
for metric in metrics:
    columns = [(node,
               metric,
              [results[density][subj][metric][node] for subj, density in output_df['Density'].iteritems()])
              for node in results[thresholds[0]][zscores.index[0]][metric].keys()]

    for node, metric, values in columns:
        output_df[f"{node}_{metric}"] = np.float64(values)

output_df.to_csv(f"{output_dir}/Graph_metrics.csv")

logging.debug(f"GRAPH METRICS COMPUTED\n{log_func(existing, locals())[0]}\n\n")
#-----------------------------------------------------------------------------------------------------------------

# Group stats

demographics = pd.read_csv(f"{data_dir}/Demographics.csv", index_col="SubjID")
gmetrics = pd.read_csv(f"{output_dir}/Graph_metrics.csv", index_col="SubjID")
metrics = gmetrics.columns.to_list()
stat_list = [np.nanmean, np.nanstd, np.nanmedian, stats.iqr, 'count']
groups = ['Dx', 'Dx3', 'Subtype', 'Minor', 'Durill3']

stat_list = ['count', np.nanvar, np.nanmean, np.nanstd, np.nanmedian, stats.iqr]
contrasts = [['Dx', 'Dx3'],
             ['Dx', 'Dx3','Minor'],
             ['Dx', 'Dx3', 'Subtype'],
             ['Dx', 'Dx3', 'Durill3']]

Ks = np.arange(gmetrics.Density.min(), gmetrics.Density.max()+1, 1)
K_ranges = np.array([(Kmin, Kmax) for Kmin in Ks for Kmax in Ks[Ks>=Kmin+3]])

for Kmin, Kmax in K_ranges:
    AUC = gmetrics[(gmetrics.Density >= Kmin) & (gmetrics.Density <= Kmax)].groupby('SubjID').sum()
    AUC = AUC.merge(demographics, left_index=True, right_index=True)

    group_stats = [AUC.groupby(groups).agg(stat_list).stack().reset_index().rename(columns={f'level_{len(groups)}':'Stat'})
                   for i, groups in enumerate(contrasts)]

    for i, data in enumerate(group_stats):
        data.sort_values(contrasts[i] + ['Stat'])
        data['Contrast'] = i

    aggregation_table = pd.concat(group_stats).set_index(['Contrast', 'Dx', 'Dx3', 'Subtype', 'Minor', 'Durill3']).reset_index()
    aggregation_table['Kmin'] = Kmin
    aggregation_table['Kmax'] = Kmax

    aggregation_table.to_csv(f"{output_dir}/Group_stats_K{Kmin}-{Kmax}.csv")

logging.debug(f"GROUP STATS COMPUTED\n{log_func(existing, locals())[0]}\n\n")

#-----------------------------------------------------------------------------------------------------------------

# Correlation coefficients

demographics = pd.read_csv(f"{data_dir}/Demographics.csv", index_col="SubjID")
gmetrics = pd.read_csv(f"{output_dir}/Graph_metrics.csv", index_col="SubjID")
metrics = gmetrics.columns.to_list()

Ks = np.arange(gmetrics.Density.min(), gmetrics.Density.max()+1, 1)
K_ranges = np.array([(Kmin, Kmax) for Kmin in Ks for Kmax in Ks[Ks>=Kmin+3]])

for Kmin, Kmax in K_ranges:
    AUC = gmetrics[(gmetrics.Density >= Kmin) & (gmetrics.Density <= Kmax)].groupby('SubjID').sum()
    AUC = AUC.merge(demographics, left_index=True, right_index=True)

    group_Rs = []
    for group in demographics.Dx3.unique():
        R = AUC[AUC.Dx3==group].corr(method='spearman')[demographics.columns]
        R.Dx3 = group
        group_Rs.append(R)

    R_table = pd.concat(group_Rs).sort_values('Dx3')
    R_table['Kmin'] = Kmin
    R_table['Kmax'] = Kmax

    R_table.to_csv(f"{output_dir}/Correlations_K{Kmin}-{Kmax}.csv")

logging.debug(f"CORRELATIONS COMPUTED\n{log_func(existing, locals())[0]}\n\n")
#-----------------------------------------------------------------------------------------------------------------

# Regression models

import statsmodels.formula.api as sm

demographics = pd.read_csv(f"{data_dir}/Demographics.csv", index_col="SubjID")
gmetrics = pd.read_csv(f"{output_dir}/Graph_metrics.csv", index_col="SubjID")[demographics.Dx3==2]
global_metrics = gmetrics.columns[gmetrics.columns.str.startswith('global')]

def modeldata(formula, data):
    model = sm.gls(formula, data=data)
    fitted = model.fit()

    endog = pd.Series('_'.join(model.endog_names.split('_')[1:]), index=['endog'])
    params = fitted.params.rename(index=dict(zip(fitted.params.index, 'param_' + fitted.params.index)))
    errors = fitted.bse.rename(index=dict(zip(fitted.params.index, 'bse_' + fitted.bse.index)))
    pvalues = fitted.pvalues.rename(index=dict(zip(fitted.pvalues.index, 'pvalues_' + fitted.bse.index)))
    rsquared = pd.Series(fitted.rsquared, index=['rsquared'])
    rsquared_adj = pd.Series(fitted.rsquared_adj, index=['rsquared_adj'])

    results = pd.concat([endog, params, errors, pvalues, rsquared, rsquared_adj])
    return results

Ks = np.arange(gmetrics.Density.min(), gmetrics.Density.max()+1, 1)
K_ranges = np.array([(Kmin, Kmax) for Kmin in Ks for Kmax in Ks[Ks>=Kmin+3]])

for Kmin, Kmax in K_ranges:
    AUC = gmetrics[(gmetrics.Density >= Kmin) & (gmetrics.Density <= Kmax)].groupby('SubjID').sum()
    AUC = AUC.merge(demographics, left_index=True, right_index=True)

    regression_results = Parallel(n_jobs=nj)(delayed
                                             (modeldata)
                                             (f"{metric} ~ Age * Bmi", data=AUC)
                                             for metric in global_metrics)

    regression_table = pd.concat([res for res in regression_results], axis=1).T.set_index('endog')

    regression_table['Kmin'] = Kmin
    regression_table['Kmax'] = Kmax

    regression_table.to_csv(f"{output_dir}/Regressions_K{Kmin}-{Kmax}.csv")

logging.debug(f"REGRESSION MODELS COMPUTED\n{log_func(existing, locals())[0]}\n\n\n\n")
#-----------------------------------------------------------------------------------------------------------------


import math, sys

def mod_pow(a, b, m): # function to calculate a^b modulo m efficiently
    result = 1 # initialize result to 1
    while b > 0: # while b is greater than 0
        if b % 2 == 1: # if b is odd
            result = (result * a) % m # multiply result by a modulo m
        a = (a * a) % m # multiply a by a modulo m
        b //= 2 # divide b by 2
    return result # return result

n = int(input()) # read the input values
for i in range(n): # for each input value
    a, b, c = map(int, input().split()) # read the input values

    # Calculate b^c modulo phi(m)
    phi = 10**9 + 6 # phi(m) = m - 1
    exp = mod_pow(b, c, phi) # calculate b^c modulo phi(m)

    # Calculate a^(b^c) modulo m using modular exponentiation
    m = 10**9 + 7 # m = 10^9 + 7
    result = mod_pow(a, exp, m) # calculate a^(b^c) modulo m
    print(result) # print the result


import os
import sys
sys.path.append(r"./")
sys.path.append(r'../../Data/')
from dataloader import Data
from Training import *
from ClassifierModels import *

print("Import Train Data...")

img_size = 256 # try 128

training_folders = [
    "../../Data/Processed/train/epidural",
    "../../Data/Processed/train/intraparenchymal",
    "../../Data/Processed/train/subarachnoid",
    "../../Data/Processed/train/intraventricular",
    "../../Data/Processed/train/subdural",
]

val_folders = [
    "../../Data/Processed/val/epidural",
    "../../Data/Processed/val/intraparenchymal",
    "../../Data/Processed/val/subarachnoid",
    "../../Data/Processed/val/intraventricular",
    "../../Data/Processed/val/subdural",
]

print("Load Alexnet data")

train_data_a = Data(training_folders,
            maximum_per_folder = 1000, #5000
            size = img_size, tl_model = "alexnet", in_channels=3,
            )

val_data_a = Data(val_folders,
            maximum_per_folder = 300, #1500
            size = img_size, tl_model = "alexnet", in_channels=3,
            )

print("Load resnet data")
train_data_r = Data(training_folders,
            maximum_per_folder = 1000, #5000
            size = img_size, tl_model = "resnet", in_channels=3,
            )

val_data_r = Data(val_folders,
            maximum_per_folder = 300, #1500
            size = img_size, tl_model = "resnet", in_channels=3,
            )

print("Amound of train data being used:", len(train_data_a))

print("Starting Alex training")
alex_model = AlexNetClassifier2(256).cuda()
alex_model.name = "classify_alex2,imgs=6k,size=256,bs=32,epochs=40,lr=0.001"
train(alex_model, train_data_a, val_data_a, batch_size=32, num_epochs=40, learning_rate=0.001, optim_param="sgd")

print("Starting Resnet training")
res_model = ResnetClass2(256).cuda()
res_model.name = "classify_res2,imgs=6k,size=256,bs=32,epochs=40,lr=0.001"
train(res_model, train_data_r, val_data_r, batch_size=32, num_epochs=40, learning_rate=0.001, optim_param="sgd")


import pygame.camera
import pygame.image
import sys
import cv2
import time
from VideoCapture import Device

cam = Device()
cam.saveSnapshot('image.jpg')
cam.setResolution(160,120)

# grab first frame
colorimg = cam.getImage()
img = colorimg.convert('L')
WIDTH = img.size[0]
HEIGHT = img.size[1]
screen = pygame.display.set_mode( ( WIDTH*2, HEIGHT*2 ) )
pygame.display.set_caption("pyGame Camera View")
average = img
diff = img

while True :
    time.sleep(0.05)
    for e in pygame.event.get() :
        if e.type == pygame.QUIT :
            sys.exit()
    rgbimg = img.convert("RGB")
    mode = rgbimg.mode
    size = rgbimg.size
    data = rgbimg.tostring()
    #print mode
    #assert mode in "RGB", "RGBA"
    surface = pygame.image.fromstring(data, size, mode)

    rgbimg = average.convert("RGB")
    mode = rgbimg.mode
    size = rgbimg.size
    data = rgbimg.tostring()
    #print mode
    #assert mode in "RGB", "RGBA"
    surface2 = pygame.image.fromstring(data, size, mode)

    rgbimg = diff.convert("RGB")
    mode = rgbimg.mode
    size = rgbimg.size
    data = rgbimg.tostring()
    #print mode
    #assert mode in "RGB", "RGBA"
    surface3 = pygame.image.fromstring(data, size, mode)

    rgbimg = colorimg
    mode = rgbimg.mode
    size = rgbimg.size
    data = rgbimg.tostring()
    #print mode
    #assert mode in "RGB", "RGBA"
    surface4 = pygame.image.fromstring(data, size, mode)

    # draw frame
    screen.blit(surface, (0,0))
    screen.blit(surface2, (WIDTH,HEIGHT))
    screen.blit(surface3, (0,HEIGHT))
    screen.blit(surface4, (WIDTH,0))

    pygame.display.flip()
    # grab next frame
    lastimg = img
    diff = lastimg
    colorimg = cam.getImage()
    img = colorimg.convert('L')
    for x in range(0, WIDTH):
        for y in range(0,HEIGHT):
            average.putpixel((x,y),(average.getpixel((x,y))*8 +  img.getpixel((x,y))*2)/10.0)

    for x in range(0, WIDTH):
        for y in range(0,HEIGHT):
            diff.putpixel( (x,y),  abs(lastimg.getpixel((x,y)) - img.getpixel((x,y))))


import os
import re
import time
from urllib import request
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#  获得某一个关键词下的所有页面的链接的url
def get_sub_page_url(browser, url):
    browser.get(url)
    time.sleep(2)
    # input_box = WebDriverWait(browser, 5).until(EC.presence_of_element_located((By.XPATH,
    #                                                                             '/html/body/div/form/div[2]/input[1]')))
    input_box = browser.find_element_by_xpath('/html/body/div/form/div[2]/input[1]')
    print(input_box)

    # 输入搜索内容
    try:
        input_box.send_keys('21世纪马克思主义')
        browser.find_element_by_xpath('/html/body/div/form/div[2]/input[2]').click()
    except:
        pass

    next = 1
    all_url = []
    while next:
        time.sleep(1)
        page_element_list = browser.find_elements_by_xpath('//td/div/a[contains(@target, "_blank")]')
        for item in page_element_list:
            all_url.append(item.get_attribute("href"))

        # 换页操作
        try:
            next_page = browser.find_element_by_link_text('下页')
            next_page.click()
            time.sleep(3)
        except:
            next_page = None
            break

    return all_url

#  获得子页的所有文本内容
def get_text_from_page(browser, all_url):
    # try:
    i = 0
    for item in all_url:
        browser.get(item)
        title = browser.find_elements_by_xpath('//h1[contains(@class, "js-title")]')
        i += 1
        print(i)
        text_page = browser.find_elements_by_xpath('//p')
        try:
            if title[0].text == '':
                with open(r'E:\21世纪马克思主义\中国青年报\{}.txt'.format(all_url[i]), 'w', encoding="utf-8") as f:
                    for it in text_page:
                        f.write('    ')
                        f.write(it.text)
                        f.write('\n')
            else:
                with open(r'E:\21世纪马克思主义\中国青年报\{}.txt'.format(title[0].text), 'w', encoding="utf-8") as f:
                    for it in text_page:
                        f.write('    ')
                        f.write(it.text)
                        f.write('\n')
        except:
            pass
        time.sleep(3)

if __name__ == "__main__":
    # 需要先print出所有页面的url后，再将url放到页面中，再查找页面内容
    url = "http://sou.cyol.com/servlet/SearchServlet.do"
    browser = webdriver.Chrome()
    options = webdriver.ChromeOptions()
    options.binary_location = r"D:\Google Chrome\Chrome\Application\chrome.exe"
    all_url = get_sub_page_url(browser, url)
    all_url = set(all_url)
    print(all_url)
    print(len(all_url))
    get_text_from_page(browser, all_url)
    browser.close()


import re
import requests

from metgem.workers.base import BaseWorker

class CheckUpdatesWorker(BaseWorker):
    URL = "https://api.github.com/repos/metgem/metgem/releases/latest"

    def __init__(self, current_version, **kwargs):
        super().__init__(**kwargs)
        self._current_version = current_version

    def run(self):
        if self.isStopped():
            self.canceled.emit()
            return False

        if not self._current_version:
            return False

        try:
            r = requests.get(CheckUpdatesWorker.URL, timeout=.5)
        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
            return False

        try:
            r.raise_for_status()
        except requests.HTTPError:
            return False

        if self.isStopped():
            self.canceled.emit()
            return False

        if r:
            json = r.json()
            tag_name = json.get("tag_name")
            if not tag_name:
                return False

            new_version = re.sub("[^0-9.]", "", tag_name)
            if not new_version or new_version <= self._current_version:
                return False

            release_notes = json.get("body")
            url = json.get("html_url")
            assets = json.get("assets")

            if not self.isStopped():
                if assets:
                    return new_version, release_notes, url
                else:
                    return False
            else:
                self.canceled.emit()


'''
Created on 2018/09/12

@author: yonezawa
'''
import subprocess
import os

def executeKallisto(kallisto_dir, seqfile, readfile_pairs, bootstrap=100):
    if trinity_util_dir[-1] != '/':
        trinity_util_dir += '/'

    calcindex_list = ['kallisto', 'index', '-i', 'region_index', seqfile]
    subprocess.call(calcindex_list)

    for i in range(len(readfile_pairs)):
        readfile1 = readfile_pairs[i][0]
        readfile2 = readfile_pairs[i][1]
        calcex_list = [kallisto_dir + 'kallisto', 'quant', '-i', 'region_index', '-o', 'condition{0:02d}_kallisto'.format(i + 1), '-b', bootstrap, readfile1, readfile2]
        subprocess.call(calcex_list)

def executeRSEM(trinity_util_dir, bowtie2_dir, seqfile, readfile_pairs, cpu=4):
    if trinity_util_dir[-1] != '/':
        trinity_util_dir += '/'
    if bowtie2_dir[-1] != '/':
        bowtie2_dir += '/'

    bowtie2_list = [bowtie2_dir + 'bowtie2-build', '-f', seqfile, seqfile]
    subprocess.call(bowtie2_list)

    for cond in readfile_pairs.keys():
        for rep in readfile_pairs[cond].keys():
            for i in range(len(readfile_pairs[cond][rep])):
                if os.path.exists(readfile_pairs[cond][rep][i]):
                    gunzip_list = ['gunzip', readfile_pairs[cond][rep][i]]
                    subprocess.call(gunzip_list)
            rsem_list = [trinity_util_dir + 'align_and_estimate_abundance.pl', '--seqType', 'fq', '--thread_count', str(cpu), '--left', readfile_pairs[cond][rep][0][:-3], '--right', readfile_pairs[cond][rep][1][:-3], '--transcripts', seqfile, '--est_method', 'RSEM', '--aln_method', 'bowtie2', '--prep_reference', '--output', 'rsem_{0:s}_{1:s}'.format(cond, rep)]
            subprocess.call(rsem_list)
            copy_list = ['cp', 'rsem_{0:s}_{1:s}/RSEM.isoforms.results'.format(cond, rep), 'rsem_{0:s}_{1:s}.dat'.format(cond, rep)]
            subprocess.call(copy_list)

    abundance_list = [trinity_util_dir + 'abundance_estimates_to_matrix.pl', '--est_method', 'RSEM', '--gene_trans_map', 'none', '--out_prefix', 'abundance_matrix']
    for cond in readfile_pairs.keys():
        for rep in readfile_pairs[cond].keys():
            abundance_list.append('rsem_{0:s}_{1:s}.dat'.format(cond, rep))

    subprocess.call(abundance_list)

if __name__ == '__main__':
    seqfile = 'focused_ranges.fasta'
    trinity_util_dir = '/usr/local/bin/trinityrnaseq-Trinity-v2.6.6/util'
    bowtie2_dir = '/home/aogura/bowtie2-2.3.4.1-linux-x86_64/'
    readfile_pairs = [['trinity_out_dir/reads.left.fq.gz.P.qtrim.gz', 'trinity_out_dir/reads.right.fq.gz.P.qtrim.gz'], ['trinity_out_dir/reads2.left.fq.gz.P.qtrim.gz', 'trinity_out_dir/reads2.right.fq.gz.P.qtrim.gz']]
    cpu = 16

    executeRSEM(trinity_util_dir, bowtie2_dir, seqfile, readfile_pairs, cpu)


from __future__ import with_statement
from builtins import range
import os
import unittest
import numpy as np
import sqlite3
import json
import tempfile
import shutil
import lsst.utils.tests
from lsst.utils import getPackageDir
from lsst.sims.utils.CodeUtilities import sims_clean_up
from lsst.sims.utils import ObservationMetaData
from lsst.sims.catalogs.db import CatalogDBObject
from lsst.sims.catalogs.definitions import InstanceCatalog
from lsst.sims.catUtils.mixins import PhotometryStars, PhotometryGalaxies
from lsst.sims.catUtils.mixins import VariabilityStars, VariabilityGalaxies
from lsst.sims.catUtils.mixins import ExtraGalacticVariabilityModels
from lsst.sims.catUtils.utils import TestVariabilityMixin

from lsst.sims.catUtils.mixins import Variability

VARIABILITY_DB = 'VariabilityTestDatabase.db'
ROOT = os.path.abspath(os.path.dirname(__file__))

def setup_module(module):
    lsst.utils.tests.init()

def makeRRlyTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the rrlyrae test
    """

    # a haphazard sample of stellar SEDs
    sedFiles = ['kp10_8750.fits_g35_8950', 'kp03_10500.fits_g45_10600', 'km50_6750.fits_g20_6750']

    # a haphazard sample of RRLyrae light curves
    lcFiles = ['rrly_lc/RRc/959802_per.txt', 'rrly_lc/RRc/1078860_per.txt', 'rrly_lc/RRab/98874_per.txt',
               'rrly_lc/RRab/3879827_per.txt']

    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE RRly
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    mjDisplacement = (rng.random_sample(size)-50.0)*50.0
    for i in range(size):
        sedFile = sedFiles[rng.randint(0, len(sedFiles))]
        varParam = {'varMethodName': 'applyRRly',
                    'pars': {'tStartMjd': 48000.0+mjDisplacement[i],
                             'filename': lcFiles[rng.randint(0, len(lcFiles))]}}
        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO RRly VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

def makeCepheidTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the cepheid test
    """

    # a haphazard sample of stellar SEDs
    sedFiles = ['kp10_8750.fits_g35_8950', 'kp03_10500.fits_g45_10600', 'km50_6750.fits_g20_6750']

    # a haphazard sample of cepheid light curves
    lcFiles = ['cepheid_lc/classical_longPer_specfile', 'cepheid_lc/classical_medPer_specfile',
               'cepheid_lc/classical_shortPer_specfile', 'cepheid_lc/classical_shortPer_specfile',
               'cepheid_lc/popII_longPer_specfile', 'cepheid_lc/popII_shortPer_specfile']

    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE cepheid
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    periods = rng.random_sample(size)*50.0
    mjDisplacement = (rng.random_sample(size)-0.5)*50.0
    for i in range(size):
        sedFile = sedFiles[rng.randint(0, len(sedFiles))]
        varParam = {'varMethodName': 'applyCepheid',
                    'pars': {'period': periods[i], 'lcfile': lcFiles[rng.randint(0, len(lcFiles))],
                             't0': 48000.0+mjDisplacement[i]}}
        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO cepheid VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

def makeEbTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the Eb test
    """

    # a haphazard sample of eclipsing binary light curves
    lcFiles = ['eb_lc/EB.2294.inp', 'eb_lc/EB.1540.inp', 'eb_lc/EB.2801.inp']

    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE eb
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    periods = rng.random_sample(size)*50.0
    mjDisplacement = (rng.random_sample(size)-0.5)*50.0
    for i in range(size):
        sedFile = 'sed_flat_norm.txt'
        varParam = {'varMethodName': 'applyEb',
                    'pars': {'period': periods[i], 'lcfile': lcFiles[rng.randint(0, len(lcFiles))],
                             't0': 48000.0+mjDisplacement[i]}}
        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO eb VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

def makeMicrolensingTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the microlensing test
    """

    # a haphazard sample of stellar SEDs
    sedFiles = ['kp10_8750.fits_g35_8950', 'kp03_10500.fits_g45_10600', 'km50_6750.fits_g20_6750']

    # there are two microlensing methods; they should be equivalent
    method = ['applyMicrolensing', 'applyMicrolens']
    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE microlensing
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    that = rng.random_sample(size)*40.0+40.0
    umin = rng.random_sample(size)
    mjDisplacement = rng.random_sample(size)*50.0
    for i in range(size):
        sedFile = sedFiles[0]
        varParam = {'varMethodName': method[i%len(method)],
                    'pars': {'that': that[i], 'umin': umin[i], 't0': 52000.0+mjDisplacement[i]}}
        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO microlensing VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

def makeBHMicrolensingTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the BHmicrolensing test
    """

    # a haphazard sample of stellar SEDs
    sedFiles = ['kp10_8750.fits_g35_8950', 'kp03_10500.fits_g45_10600', 'km50_6750.fits_g20_6750']

    # a sample of black hole microlensing light curves that do not repeat time steps
    # (repeating time steps causes the scipy spline interpolation routine to return Nan)
    lcFiles = ['microlens/bh_binary_source/lc_14_25_75_8000_0_0.05_316',
               'microlens/bh_binary_source/lc_14_25_4000_8000_0_phi1.09_0.005_100',
               'microlens/bh_binary_source/lc_14_25_75_8000_0_tets2.09_0.005_316']

    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE bhmicrolensing
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    mjDisplacement = rng.random_sample(size)*5.0*365.25
    for i in range(size):
        sedFile = sedFiles[rng.randint(0, len(sedFiles))]
        varParam = {'varMethodName': 'applyBHMicrolens',
                    'pars': {'filename': lcFiles[rng.randint(0, len(lcFiles))],
                             't0': 52000.0-mjDisplacement[i]}}
        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO bhmicrolensing VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

def makeAmcvnTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the AMCVN test
    """

    # a haphazard sample of white dwarf SEDs
    sedFiles = ['bergeron_He_4750_70.dat_4950', 'bergeron_50000_85.dat_54000']

    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE amcvn
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    doesBurst = rng.randint(0, 2, size=size)
    burst_freq = rng.randint(10, 150, size=size)
    burst_scale = 115.0
    amp_burst = rng.random_sample(size)*8.0
    color_excess_during_burst = rng.random_sample(size)*0.2-0.4
    amplitude = rng.random_sample(size)*0.2
    period = rng.random_sample(size)*200.0
    mjDisplacement = rng.random_sample(size)*500.0
    for i in range(size):
        sedFile = sedFiles[rng.randint(0, len(sedFiles))]
        varParam = {'varMethodName': 'applyAmcvn',
                    'pars': {'does_burst': int(doesBurst[i]),  # have to cast to int from np.int for json
                             'burst_freq': int(burst_freq[i]),
                             'burst_scale': burst_scale,
                             'amp_burst': amp_burst[i],
                             'color_excess_during_burst': color_excess_during_burst[i],
                             'amplitude': amplitude[i],
                             'period': period[i],
                             't0': 51500.0-mjDisplacement[i]}}

        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO amcvn VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

def makeAgnTable(size=100, database=VARIABILITY_DB, **kwargs):
    """
    Make a test database to serve information to the microlensing test
    """

    # a haphazard sample of galaxy SEDs
    sedFiles = ['Exp.31E06.0005Z.spec', 'Inst.79E06.1Z.spec', 'Const.50E07.0005Z.spec']
    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE agn
                     (galid int, varsimobjid int,
                      internalAvBulge real, internalAvDisk real, redshift real,
                      variability text,
                      sedFilenameBulge text, sedFilenameDisk text, sedFilenameAgn text)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    agn_tau = rng.random_sample(size)*100.0+100.0
    agn_sfu = rng.random_sample(size)*2.0
    agn_sfg = rng.random_sample(size)*2.0
    agn_sfr = rng.random_sample(size)*2.0
    agn_sfi = rng.random_sample(size)*2.0
    agn_sfz = rng.random_sample(size)*2.0
    agn_sfy = rng.random_sample(size)*2.0
    mjDisplacement = rng.random_sample(size)*5.0
    avBulge = rng.random_sample(size)*0.5+2.6
    avDisk = rng.random_sample(size)*0.5+2.6
    redshift = rng.random_sample(size)*0.5
    for i in range(size):
        varParam = {'varMethodName': 'applyAgn',
                    'pars': {'agn_tau': agn_tau[i], 'agn_sfu': agn_sfu[i], 'agn_sfg': agn_sfg[i],
                             'agn_sfr': agn_sfr[i], 'agn_sfi': agn_sfi[i], 'agn_sfz': agn_sfz[i],
                             'agn_sfy': agn_sfy[i], 't0_mjd': 48000.0+mjDisplacement[i],
                             'seed': rng.randint(0, 200000)}}

        paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO agn VALUES (%i, %i, %f, %f, %f, '%s', '%s', '%s', '%s')''' % \
               (i, i, avBulge[i], avDisk[i], redshift[i],
                paramStr,
                sedFiles[rng.randint(0, len(sedFiles))],
                sedFiles[rng.randint(0, len(sedFiles))],
                'agn.spec')

        c.execute(qstr)
    conn.commit()
    conn.close()

def makeHybridTable(size=100, database='VariabilityTestDatabase.db', **kwargs):
    """
    Make a test database that contains a mix of Cepheid variables
    and 'testVar' variables (variables that use the applySineVar
    method defined in the TestVariabilityMixin)
    """

    # a haphazard sample of stellar SEDs
    sedFiles = ['kp10_8750.fits_g35_8950', 'kp03_10500.fits_g45_10600', 'km50_6750.fits_g20_6750']

    # a haphazard sample of cepheid light curves
    lcFiles = ['cepheid_lc/classical_longPer_specfile', 'cepheid_lc/classical_medPer_specfile',
               'cepheid_lc/classical_shortPer_specfile', 'cepheid_lc/classical_shortPer_specfile',
               'cepheid_lc/popII_longPer_specfile', 'cepheid_lc/popII_shortPer_specfile']

    conn = sqlite3.connect(database)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE hybrid
                     (varsimobjid int, variability text, sedfilename text, parallax real, ebv real)''')
        conn.commit()
    except:
        return

    rng = np.random.RandomState(32)
    periods = rng.random_sample(size)*50.0
    mjDisplacement = (rng.random_sample(size)-0.5)*50.0
    for i in range(size):
        sedFile = sedFiles[rng.randint(0, len(sedFiles))]
        if i%3 == 0:
            # just to make sure that Variability mixins no how to andle
            # objects with no variability
            varParam = None
            paramStr = None
        elif i%2 == 0:
            varParam = {'varMethodName': 'applyCepheid',
                        'pars': {'period': periods[i],
                                 'lcfile': lcFiles[rng.randint(0, len(lcFiles))],
                                 't0': 48000.0+mjDisplacement[i]}}
        else:
            varParam = {'varMethodName': 'testVar',
                        'pars': {'period': rng.random_sample()*100.0, 'amplitude': 2.0}}

        if varParam is not None:
            paramStr = json.dumps(varParam)

        qstr = '''INSERT INTO hybrid VALUES (%i, '%s', '%s', 0.01, 0.7)''' % (i, paramStr, sedFile)
        c.execute(qstr)
    conn.commit()
    conn.close()

class variabilityDB(CatalogDBObject):
    driver = 'sqlite'
    database = VARIABILITY_DB
    idColKey = 'varsimobjid'
    columns = [('id', 'varsimobjid', int),
               ('sedFilename', 'sedfilename', str, 40),
               ('varParamStr', 'variability', str, 600)]

class hybridDB(variabilityDB):
    objid = 'hybridTest'
    tableid = 'hybrid'
    objectTypeId = 54

class rrlyDB(variabilityDB):
    objid = 'rrlyTest'
    tableid = 'RRly'
    objectTypeId = 55

class cepheidDB(variabilityDB):
    objid = 'cepheidTest'
    tableid = 'cepheid'
    objectTypeId = 56

class ebDB(variabilityDB):
    objid = 'ebTest'
    tableid = 'eb'
    objectTypeId = 57

class microlensDB(variabilityDB):
    objid = 'microlensTest'
    tableid = 'microlensing'
    objectTypeId = 58

class BHmicrolensDB(variabilityDB):
    objid = 'bhmicrolensTest'
    tableid = 'bhmicrolensing'
    objectTypeId = 59

class amcvnDB(variabilityDB):
    objid = 'amcvnTest'
    tableid = 'amcvn'
    objectTypeId = 60

class agnDB(variabilityDB):
    objid = 'agnTest'
    tableid = 'agn'
    objectTypeId = 61

class StellarVariabilityCatalog(InstanceCatalog, PhotometryStars, VariabilityStars):
    catalog_type = __file__ + 'stellarVariabilityCatalog'
    column_outputs = ['varsimobjid', 'sedFilename', 'delta_lsst_u']
    default_columns = [('magNorm', 14.0, float)]

class StellarVariabilityCatalogWithTest(InstanceCatalog, PhotometryStars,
                                        VariabilityStars, TestVariabilityMixin):
    catalog_type = __file__ + 'testVariabilityCatalog'
    column_outputs = ['varsimobjid', 'sedFilename', 'delta_lsst_u']
    default_columns = [('magNorm', 14.0, float)]

class OtherVariabilityCatalogWithTest(InstanceCatalog, PhotometryStars,
                                      TestVariabilityMixin, VariabilityStars):
    catalog_type = __file__ + 'other_variability_catalog'
    column_outputs = ['varsimobjid', 'sedFilename', 'delta_lsst_u']
    default_columns = [('magNorm', 14.0, float)]

class GalaxyVariabilityCatalog(InstanceCatalog, PhotometryGalaxies, VariabilityGalaxies):
    catalog_type = __file__ + 'galaxyVariabilityCatalog'
    column_outputs = ['varsimobjid', 'sedFilenameAgn', 'lsstUdiff', 'delta_uAgn']
    default_columns = [('magNormAgn', 14.0, float),
                       ('magNormDisk', 14.0, float),
                       ('magNormBulge', 14.0, float)]

    def get_lsstUdiff(self):
        lsstUvar = self.column_by_name('lsst_u')

        bulge = self.column_by_name('uBulge')
        disk = self.column_by_name('uDisk')
        agn = self.column_by_name('uAgn') - self.column_by_name('delta_uAgn')
        lsstU = self.sum_magnitudes(bulge=bulge, disk=disk, agn=agn)

        return lsstUvar - lsstU

    def get_agnUdiff(self):
        lsstU = self.column_by_name('uAgn')
        lsstUvar = self.column_by_name('uAgn_var')
        return lsstUvar - lsstU

class VariabilityTest(unittest.TestCase):

    longMessage = True

    @classmethod
    def setUpClass(cls):
        cls.scratch_dir = tempfile.mkdtemp(dir=ROOT, prefix='VariabilityTest-')
        cls.variability_db = os.path.join(cls.scratch_dir, VARIABILITY_DB)

    @classmethod
    def tearDownClass(cls):
        sims_clean_up()
        if os.path.exists(cls.variability_db):
            os.unlink(cls.variability_db)
        if os.path.exists(cls.scratch_dir):
            shutil.rmtree(cls.scratch_dir, ignore_errors=True)

    def setUp(self):
        self.obs_metadata = ObservationMetaData(mjd=52000.0)

    def tearDown(self):
        del self.obs_metadata

    def verify_catalogs(self, cat_name):
        """
        Verify that a catalog generated by the unit tests below contains
        the rows it ought to.

        This is done by looking for a corresponding catalog in
        tests/testData and verifying that the two catalogs have identical
        rows.

        Parameters
        ----------
        cat_name is the full path to the test-generated catalog.  The
        comparison catalog will be found in tests/testData
        """

        control_dir = os.path.join(getPackageDir('sims_catUtils'),
                                   'tests', 'testData')
        _, control_name = os.path.split(cat_name)
        control_name = os.path.join(control_dir, control_name)
        with open(control_name, 'r') as control_file:
            control_lines = control_file.readlines()

        with open(cat_name, 'r') as test_file:
            test_lines = test_file.readlines()

        for tt in test_lines:
            self.assertIn(tt, control_lines)

        for cc in control_lines:
            self.assertIn(cc, test_lines)

    def testHybridVariability(self):
        """
        Test that we can generate a catalog which inherits from multiple variability mixins
        (in this case, TestVariability and VariabilityStars).  This is to make sure that
        the register_method and register_class decorators do not mangle inheritance of
        methods from mixins.
        """
        cat_name = os.path.join(self.scratch_dir, 'hybridTestCatalog.dat')
        makeHybridTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('hybridTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalogWithTest(myDB, obs_metadata=self.obs_metadata)

        myCatalog.write_catalog(cat_name, chunk_size=1000)
        self.verify_catalogs(cat_name)

        if os.path.exists(cat_name):
            os.unlink(cat_name)

        # make sure order of mixin inheritance does not matter
        myCatalog = OtherVariabilityCatalogWithTest(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)
        self.verify_catalogs(cat_name)

        if os.path.exists(cat_name):
            os.unlink(cat_name)

        # make sure that, if a catalog does not contain a variability method,
        # an error is thrown; verify that it contains the correct error message
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)

        with self.assertRaises(RuntimeError) as context:
            myCatalog.write_catalog(cat_name)

        if os.path.exists(cat_name):
            os.unlink(cat_name)

        expectedMessage = "Your InstanceCatalog does not contain a variability method"
        expectedMessage += " corresponding to 'testVar'"
        self.assertEqual(context.exception.args[0], expectedMessage)

    def testApplyVariabilityWithManyMJD(self):
        """
        Use the hybrid variability catalog class from testHybridVariability to
        verify that, if we pass an array of expmjd into applyVariability, we get
        the expected result out.
        """

        makeHybridTable(database=self.variability_db)
        hybrid_db = hybridDB(database=self.variability_db)
        hybrid_cat = StellarVariabilityCatalogWithTest(hybrid_db, obs_metadata=self.obs_metadata,
                                                       column_outputs=['varParamStr'])
        rng = np.random.RandomState(88)
        mjd_arr = rng.random_sample(14)*3653.0+59580.0
        n_time = len(mjd_arr)
        varparams =[]
        n_ceph = 0
        n_test = 0
        for line in hybrid_cat.iter_catalog():
            varparams.append(line[-1])
            if 'Cepheid' in line[-1]:
                n_ceph += 1
            if 'testVar' in line[-1]:
                n_test +=1

        self.assertGreater(n_ceph, 0)
        self.assertGreater(n_test, 0)

        n_obj=len(varparams)

        new_hybrid_cat = StellarVariabilityCatalogWithTest(hybrid_db, obs_metadata=self.obs_metadata)
        delta_mag_vector = new_hybrid_cat.applyVariability(varparams, expmjd=mjd_arr)
        self.assertEqual(delta_mag_vector.shape, (6, len(varparams), len(mjd_arr)))

        n_mags = 0
        for i_time, mjd in enumerate(mjd_arr):
            obs = ObservationMetaData(mjd=mjd)
            control_hybrid_cat = StellarVariabilityCatalogWithTest(hybrid_db, obs_metadata=obs,
                                                                   column_outputs=['delta_lsst_u',
                                                                                'delta_lsst_g',
                                                                                'delta_lsst_r',
                                                                                'delta_lsst_i',
                                                                                'delta_lsst_z',
                                                                                'delta_lsst_y'])

            for i_obj, star_obj in enumerate(control_hybrid_cat.iter_catalog()):
                for i_band in range(6):
                    n_mags += 1
                    self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
                                     star_obj[-6+i_band],
                                     msg = 'failed on time %d; obj %d; band %d' % (i_time, i_obj, i_band))

        self.assertEqual(n_mags, 6*n_obj*n_time)

    def testRRlyrae(self):
        cat_name = os.path.join(self.scratch_dir, 'rrlyTestCatalog.dat')
        makeRRlyTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('rrlyTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        self.verify_catalogs(cat_name)
        if os.path.exists(cat_name):
            os.unlink(cat_name)

    def testCepheids(self):
        cat_name = os.path.join(self.scratch_dir, 'cepheidTestCatalog.dat')
        makeCepheidTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('cepheidTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        self.verify_catalogs(cat_name)
        if os.path.exists(cat_name):
            os.unlink(cat_name)

    def testEb(self):
        cat_name = os.path.join(self.scratch_dir, 'ebTestCatalog.dat')
        makeEbTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('ebTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        self.verify_catalogs(cat_name)
        if os.path.exists(cat_name):
            os.unlink(cat_name)

    def testMicrolensing(self):
        # Note:  this test assumes that the parameters for the microlensing variability
        # model occur in a standard varParamStr column in the database.
        # Actually, the current database of microlensing events simply store the variability
        # parameters as independent columns in the database.
        # The varParamStr formalism is how the applyMicrolensing methods are written, however,
        # so that is what we will test.

        cat_name = os.path.join(self.scratch_dir, 'microlensTestCatalog.dat')
        makeMicrolensingTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('microlensTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        self.verify_catalogs(cat_name)
        if os.path.exists(cat_name):
            os.unlink(cat_name)

    def testBHMicrolensing(self):
        # Note:  this test assumes that the parameters for the BHmicrolensing variability
        # model occur in a standard varParamStr column in the database.
        # Actually, the current database of BHmicrolensing events simply store the variability
        # parameters as independent columns in the database.
        # The varParamStr formalism is how the applyBHMicrolens method is written, however,
        # so that is what we will test.

        cat_name = os.path.join(self.scratch_dir, 'bhmicrolensTestCatalog.dat')
        makeBHMicrolensingTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('bhmicrolensTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        self.verify_catalogs(cat_name)
        if os.path.exists(cat_name):
            os.unlink(cat_name)

    def testAmcvn(self):
        # Note:  this test assumes that the parameters for the Amcvn variability
        # model occur in a standard varParamStr column in the database.
        # Actually, the current database of Amcvn events simply store the variability
        # parameters as independent columns in the database.
        # The varParamStr formalism is how the applyAmcvn method is written, however,
        # so that is what we will test.

        cat_name = os.path.join(self.scratch_dir, 'amcvnTestCatalog.dat')
        makeAmcvnTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('amcvnTest', database=self.variability_db)
        myCatalog = StellarVariabilityCatalog(myDB, obs_metadata=self.obs_metadata)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        self.verify_catalogs(cat_name)
        if os.path.exists(cat_name):
            os.unlink(cat_name)

    def testAgn(self):
        """
        Just verify that the catalog generation code runs in this case
        """

        cat_name = os.path.join(self.scratch_dir, 'agnTestCatalog.dat')
        makeAgnTable(database=self.variability_db)
        myDB = CatalogDBObject.from_objid('agnTest', database=self.variability_db)
        obs = ObservationMetaData(pointingRA=self.obs_metadata.pointingRA,
                                  pointingDec=self.obs_metadata.pointingDec,
                                  boundType=self.obs_metadata.boundType,
                                  boundLength=self.obs_metadata.boundLength,
                                  mjd=60000.0)
        myCatalog = GalaxyVariabilityCatalog(myDB, obs_metadata=obs)
        myCatalog.write_catalog(cat_name, chunk_size=1000)

        with open(cat_name,'r') as input_file:
            lines = input_file.readlines()
            self.assertGreater(len(lines), 10)

        if os.path.exists(cat_name):
            os.unlink(cat_name)

class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
    pass

if __name__ == "__main__":
    lsst.utils.tests.init()
    unittest.main()