from typing import List

import torch
from torch import optim

from trajpred.config import Config

from torch.optim.lr_scheduler import MultiStepLR

class Optimizer(torch.optim.Optimizer):
    def __init__(self, params, config: Config, coef=None):

        self.config = config
        if not (isinstance(params, list) or isinstance(params, tuple)):
            params = [params]

        if coef is None:
            coef = [1.0] * len(params)
        else:
            if isinstance(coef, list) or isinstance(coef, tuple):
                assert len(coef) == len(params)
            else:
                coef = [coef] * len(params)
        self.coef = coef

        param_groups = []
        for param in params:
            # param_groups.append({"params": param, "lr": 0})
            param_groups.append({"params": param, "lr": config.hydra.model.train.learning_rate})

        opt = config.hydra.model.train.optimizer.lower()
        assert opt == "adam" # it only supports adam. you can add a new optimizer by adding to the config file
        # self.opt = optim.Adam(param_groups, weight_decay=0)
        # self.opt = optim.Adam(param_groups, lr=config.hydra.train.learning_rate, eps=0.0001)
        self.opt = optim.Adam(param_groups, eps=0.0001)

        # self.lr_func = StepLR(config.hydra.train.lrs, config.hydra.train.lr_step_iters) #config.lr_func
        if config.hydra.model.train.scheduler == "multistep":
            self.lr_func = MultiStepLR(self.opt, milestones=config.hydra.model.train.learning_rate_sched, gamma=0.5,
                                               verbose=True)
        elif config.hydra.model.train.scheduler == "plateau":
            self.lr_func = torch.optim.lr_scheduler.ReduceLROnPlateau(self.opt, mode='min', factor=0.5, patience=5)
        else:
            raise NotImplementedError

        if hasattr(config, "clip_grads"):
            self.clip_grads = config["clip_grads"]
            self.clip_low = config["clip_low"]
            self.clip_high = config["clip_high"]
        else:
            self.clip_grads = False

    def zero_grad(self):
        self.opt.zero_grad()

    def step(self):
        if self.clip_grads:
            self.clip()
        self.opt.step()

        # From Autobot
        lr = self.opt.param_groups[0]['lr']

        return lr
        # return self.opt.step()

    def scheduler_step(self, val_metric=0):
        if self.config.hydra.model.train.scheduler == "multistep":
            return self.lr_func.step()
        else:
            return self.lr_func.step(val_metric)

    def state_dict(self) -> dict:
        return self.opt.state_dict()

    def clip(self):
        low, high = self.clip_low, self.clip_high
        params = []
        for param_group in self.opt.param_groups:
            params += list(filter(lambda p: p.grad is not None, param_group["params"]))
        for p in params:
            mask = p.grad.data < low
            p.grad.data[mask] = low
            mask = p.grad.data > high
            p.grad.data[mask] = high

    def load_state_dict(self, opt_state):
        self.opt.load_state_dict(opt_state)

class StepLR:
    def __init__(self, lrs: List[float], step_iters: List[int]) -> None:
        assert (len(lrs) == len(step_iters) + 1)
        self._lrs = lrs
        self._step_iters = step_iters
        self._iter = 0

    def __call__(self):
        self._iter += 1
        for i, step_iter in enumerate(self._step_iters):
            if self._iter % 1000 == 0:
                print("INFO: _iter", self._iter, step_iter)
            if self._iter <= step_iter:
                return self._lrs[i]
        return self._lrs[-1]


#!/usr/bin/env python
#-*- coding: utf-8 -*-
################################################################################
#                     - EMULATIONSTATION GAMELIST PATCH -                      #
#                                - MERGE XML -                                 #
#------------------------------------------------------------------------------#
# NORDIC POWER amiga15@outlook.fr                 0.2.0  15/05/2017-02/07/2017 #
#------------------------------------------------------------------------------#
#0.2.0 - 02/07/2017 - Support des folders                                      #
################################################################################

#IMPORT STD---------------------------------------------------------------------
import os.path
import sys
import time
import shutil
import traceback
import argparse
from os.path import basename, splitext

#IMPORT NORDICPOWER--------------------------------------------------------------
from GameListXml import *
from GameListDir import *
from GameListTools import *
from GameListPatch import *

#CONSTANTS-----------------------------------------------------------------------
VERSION='0.2.0 BETA'

#CONSTANTS ARGUMENTS LANCEMENT---------------------------------------------------
ARG_FILE_SRC='file_src'
ARG_FILE_DEST='file_dest'

#MSG LOCALISATION----------------------------------------------------------------
config=Config()

msg_local={
('MSG_ERROR_MX_EXCEPTION','FR'):u'{} : {}',
('MSG_ERROR_MX_EXCEPTION','EN'):u'{} : {}',
('MSG_ERROR_MX_GAMEXML_SRC_NOT_FOUND','FR'):u'Fichier {} non trouv\u00E9 !',
('MSG_ERROR_MX_GAMEXML_SRC_NOT_FOUND','EN'):u'file {} not found !',
('MSG_INFO_MX_GAMEXML_DEST_NOT_FOUND','FR'):u'Fichier dest {} non trouv\u00E9, copie du fichier source',
('MSG_INFO_MX_GAMEXML_DEST_NOT_FOUND','EN'):u'destination file {} not found, make a simple copy',
('MSG_ERROR_MX_GAMEXML_DEST_COPY','FR'):u'Erreur dans la copie',
('MSG_ERROR_MX_GAMEXML_DEST_COPY','EN'):u'error during the copy',
('MSG_ERROR_MX_LOADING_GAMEXML','FR'):u'Erreur dans le fichier xml {}',
('MSG_ERROR_MX_LOADING_GAMEXML','EN'):u'xml error in {}',
('MSG_INFO_MX_PRG_END','FR'):u'Fusion termin\u00E9',
('MSG_INFO_MX_PRG_END','EN'):u'Merge end'
}

#---------------------------------------------------------------------------------------------------
def get_args():
	parser = argparse.ArgumentParser(description='fusion de fichiers gamelist.xml',epilog='(C) NORDIC POWER')
	parser.add_argument(ARG_FILE_SRC)
	parser.add_argument(ARG_FILE_DEST)
	return parser.parse_args()

#---------------------------------------------------------------------------------------------------
#---------------------------------------- MAIN -----------------------------------------------------
#---------------------------------------------------------------------------------------------------
def main():

	#Test Arg
	args=get_args()

	#Init Log
	logger=Logger.logr
	Logger.setLevel(logging.DEBUG)
	Logger.add_handler_console()

	logger.info('mergexml - NordicPower - Version '+VERSION)
	#Lecture Configuration
	config.load_from_file()

	if not os.path.isfile(args.file_src):
		logger.error(msg_local.get(('MSG_ERROR_MX_GAMEXML_SRC_NOT_FOUND',config.language)).format(args.file_src))
		sys.exit(1)

	#Chargement du fichier source
	gamesList_src  = GamesList()
	try:
		gamesList_src.import_xml_file(args.file_src)
	except MyError:
		logger.error(msg_local.get(('MSG_ERROR_MX_LOADING_GAMEXML',config.language)).format(args.file_src))
		sys.exit(1)

	#Pas de fichier destination, réalisation d'une simple copie de fichiers
	if not os.path.isfile(args.file_dest):
		try:
			shutil.copyfile(args.file_src, args.file_dest)
			logger.error(msg_local.get(('MSG_INFO_MX_GAMEXML_DEST_NOT_FOUND',config.language)).format(args.file_dest))
		except:
			pass
		sys.exit(0)

	#Chargement du fichier destination
	gamesList_dest = GamesList()
	try:
		gamesList_dest.import_xml_file(args.file_dest)
	except MyError:
		logger.error(msg_local.get(('MSG_ERROR_MX_LOADING_GAMEXML',config.language)).format(args.file_dest))
		sys.exit(1)

	#Fusion des XML
	for game_XML_src in gamesList_src.get_games():

		try:
			game_XML_dest = gamesList_dest.search_game_by_path(game_XML_src.path)

			#Game trouvé, la source écrase les infos de la destination
			game_XML_dest.path = game_XML_src.path
			if game_XML_src.name!='':game_XML_dest.name = game_XML_src.name
			if game_XML_src.desc!='':game_XML_dest.desc = game_XML_src.desc
			if game_XML_src.image!='':game_XML_dest.image = game_XML_src.image
			if game_XML_src.rating!='':game_XML_dest.rating = game_XML_src.rating
			if game_XML_src.releasedate!='':game_XML_dest.releasedate = game_XML_src.releasedate
			if game_XML_src.developer!='':game_XML_dest.developer = game_XML_src.developer
			if game_XML_src.publisher!='':game_XML_dest.publisher = game_XML_src.publisher
			if game_XML_src.genre!='':game_XML_dest.genre = game_XML_src.genre
			if game_XML_src.players!='':game_XML_dest.players = game_XML_src.players
			if game_XML_src.ref_id!='':game_XML_dest.ref_id = game_XML_src.ref_id
			if game_XML_src.source!='':game_XML_dest.source = game_XML_src.source
			if game_XML_src.playcount!='':game_XML_dest.playcount = game_XML_src.playcount
			if game_XML_src.lastplayed!='':game_XML_dest.lastplayed = game_XML_src.lastplayed

			gamesList_dest.update_game(game_XML_dest)

		except MyError:
			#Game non trouvé, simple recopie
			game_XML_dest = Game()
			game_XML_dest.path = game_XML_src.path
			game_XML_dest.copy_from_game(game_XML_src)

			gamesList_dest.add_game(game_XML_dest)

	#Fusion des XML
	for folder_XML_src in gamesList_src.get_folders():

		try:
			folder_XML_dest = gamesList_dest.search_folder_by_path(folder_XML_src.path)

			#folder trouvé, la source écrase les infos de la destination
			folder_XML_dest.path = folder_XML_src.path
			if folder_XML_src.name!='':folder_XML_dest.name = folder_XML_src.name
			if folder_XML_src.desc!='':folder_XML_dest.desc = folder_XML_src.desc
			if folder_XML_src.image!='':folder_XML_dest.image = folder_XML_src.image
			if folder_XML_src.hidden!='':folder_XML_dest.hidden = folder_XML_src.hidden

			gamesList_dest.update_folder(folder_XML_dest)

		except MyError:
			#folder non trouvé, simple recopie
			folder_XML_dest = Folder()
			folder_XML_dest.path = folder_XML_src.path
			folder_XML_dest.name = folder_XML_src.name
			folder_XML_dest.desc = folder_XML_src.desc
			folder_XML_dest.image = folder_XML_src.image
			folder_XML_dest.hidden = folder_XML_src.hidden
			gamesList_dest.add_folder(folder_XML_dest)

	#STEP END - Sauvegarde si intervention sur le fichier--------------------------------------------
	if gamesList_dest.modified:

		if len(gamesList_dest.get_games())>SEUIL_SAUVEGARDE_BIGROM:
			#logger.debug(msg_local.get(('MSG_INFO_GLP_PRG_SAVE_BIGROM',config.language)).format(args.file_dest))
			gamesList_dest.save_xml_file(args.file_dest,False)
		else:
			#logger.debug(msg_local.get(('MSG_INFO_GLP_PRG_SAVE',config.language)).format(args.file_dest))
			gamesList_dest.save_xml_file(args.file_dest)

	logger.info(msg_local.get(('MSG_INFO_MX_PRG_END',config.language)))

	#Fin
	sys.exit(0)

#---------------------------------------------------------------------------------------------------
if __name__ == '__main__':
	main()


import requests
from dotenv import load_dotenv
import boto3
from boto3.dynamodb.conditions import Key
import botocore

from pprint import pprint
import urllib
import os
import base64
import json

from typing import List
from datetime import datetime, timedelta

load_dotenv()

class Tweet:
    def __init__(self, data):
        self.id: str = data['id']
        self.text: str = data['text']
        self.date: datetime = from_iso8601(data['created_at'])
        self.sent = data.get('text_sent', False)

    def __str__(self):
        return f'created at {self.date}'

class DogResource():
    def __init__(self, dynamodb=None):
        self._dynamodb = get_dynamodb(dynamodb)
        self._table = get_table('Dog')

    def create_table(self):
        self._table = self._dynamodb.create_table(
            TableName='Dog',
            KeySchema=[
                {
                    'AttributeName': 'tweet_id',
                    'KeyType': 'HASH'
                },
                {
                    'AttributeName': 'created_at',
                    'KeyType': 'RANGE'
                }
            ],
            AttributeDefinitions=[
                {
                    'AttributeName': 'tweet_id',
                    'AttributeType': 'S'
                },
                {
                    'AttributeName': 'created_at',
                    'AttributeType': 'S'
                },
            ],
            ProvisionedThroughput={
                'ReadCapacityUnits': 10,
                'WriteCapacityUnits': 10
            }
        )

    def clean(self):
        try:
            self._table.delete()
        except Exception:
            pass
        self.create_table()

    def put(self, tweet: Tweet):
        return self._table.put_item(Item={
            'tweet_id': tweet.id,
            'created_at': tweet.date.isoformat(),
            'text': tweet.text,
            'text_sent': tweet.sent
        })

    def update(self, tweet: Tweet):
        return self._table.update_item(
            Key={
                'tweet_id': tweet.id,
                'created_at': tweet.date.isoformat()
            },
            UpdateExpression="set text_sent=:s",
            ExpressionAttributeValues={':s': tweet.sent},
            ReturnValues='UPDATED_NEW'
        )

    def get(self, tweet: Tweet):
        response = self._table.get_item(Key={
            'tweet_id': tweet.id,
            'created_at': tweet.date.isoformat()
        })
        return response.get('Item', None)

    def get_since(self, dt: datetime):
        response = self._table.scan(
            FilterExpression=Key('created_at').gt(dt.isoformat()),
            ProjectionExpression='tweet_id, created_at, #t, text_sent',
            ExpressionAttributeNames={'#t': 'text'}
        )
        data = [{
            'id': item['tweet_id'],
            'created_at': item['created_at'],
            'text': item['text'],
            'text_sent': item['text_sent']
        } for item in response['Items']]

        return [Tweet(item) for item in data]

def get_dog_tweets() -> List[Tweet]:
    api_key = os.getenv('TWITTER_API_KEY')
    api_key_secret = os.getenv('TWITTER_API_KEY_SECRET')
    bearer_token = os.getenv('TWITTER_BEARER_TOKEN')
    user_id = '846137120209190912'
    url = f'https://api.twitter.com/2/users/{user_id}/tweets'
    params = {
        'tweet.fields': 'created_at',
        'max_results': 20,
    }
    headers = {
        'Authorization': f'Bearer {bearer_token}',
        'Accept': 'application/json'
    }

    print(f'sending request to {url}')
    r = requests.get(
            url,
            params=params,
            headers=headers
    )
    print(f'got response {r}')

    return [Tweet(item) for item in r.json()['data']]

def send_text(phone_number: str, message: str):
    auth_token = os.getenv('TWILIO_AUTH_TOKEN')
    account_sid = 'ACa0ce39a5b490614bc46ada6956b0a64c'

    headers = {
        'Authorization': 'Basic ' + base64.b64encode(f'{account_sid}:{auth_token}'.encode('utf-8')).decode('utf8')
    }

    data = {
        'To': phone_number,
        'MessagingServiceSid': 'MG8438fe356d5320990608ed4940daa10c',
        'Body': message
    }

    print('sending twilio post request')
    r = requests.post(
        'https://api.twilio.com/2010-04-01/Accounts/ACa0ce39a5b490614bc46ada6956b0a64c/Messages.json',
        data = data,
        headers = headers
    )
    print(f'got response {r}')

def from_iso8601(s):
    if s[-1].lower() == 'z':
        return datetime.fromisoformat(s[:-1])
    return datetime.fromisoformat(s)

def create_dog_table(dynamodb=None):
    dynamodb = get_dynamodb(dynamodb)

    table = dynamodb.create_table(
        TableName='Dog',
        KeySchema=[
            {
                'AttributeName': 'tweet_id',
                'KeyType': 'HASH'
            },
            {
                'AttributeName': 'created_at',
                'KeyType': 'RANGE'
            }
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'tweet_id',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'created_at',
                'AttributeType': 'S'
            },
        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 10,
            'WriteCapacityUnits': 10
        }
    )

    return table

def load_test_data():
    with open('example_twitter_response.json') as f:
        return [Tweet(item) for item in json.load(f)['data']]

def get_dynamodb(dynamodb=None):
    if dynamodb:
        return dynamodb

    endpoint_url = os.getenv('DYNAMODB_URL')
    if endpoint_url:
        return boto3.resource('dynamodb', endpoint_url=os.getenv('DYNAMODB_URL'))
    return boto3.resource('dynamodb', region_name='us-east-1')

def get_table(table_name, dynamodb=None):
    dynamodb = get_dynamodb(dynamodb)
    return dynamodb.Table(table_name)

def good_tweet(tweet):
    # currently removing the expired requiement
    # expired = tweet.date > datetime.now() - timedelta(days = 7)
    expired = tweet.date < datetime.strptime('2021/11/01', '%Y/%m/%d')
    retweet = tweet.text.startswith("RT")
    contains_link = ".com" in tweet.text or ".org" in tweet.text or "http" in tweet.text
    contains_at = "@" in tweet.text
    print(f'expired={expired}, retweet={retweet}, contains_link={contains_link}, contians_at={contains_at}')
    return not expired and not retweet and not contains_link and not contains_at

def main(event=None, context=None):
    dog_resource = DogResource()

    # fetch tweets and add to database
    tweets = get_dog_tweets()
    for tweet in tweets:
        print(f'tweet text: {tweet.text}')
        good = good_tweet(tweet)
        present = (not dog_resource.get(tweet) is None)
        print(f'good={good}')
        print(f'present={present}')
        if good and not present:
            if os.getenv('TEST'):
                print('skipping database put due to TEST environment variable')
                continue
            dog_resource.put(tweet)

    # send texts for tweets which have not been sent
    sent_count = 0
    for tweet in dog_resource.get_since(datetime.now() - timedelta(7)):
        if not tweet.sent:
            print(f'sending text for tweet {tweet.date}')
            if os.getenv('TEST') == "TRUE":
                print('skipping sms and database update, due to TEST environment variable')
                continue
            send_text(os.getenv('TO_PHONE'), f'{tweet.text}\n\n- Dog')
            tweet.sent = True
            dog_resource.update(tweet)

            # we only want to send one tweet per run
            break

if __name__ == '__main__':
    main()


"""
Studio editing view for Audio XBlock
"""

import json

from xblock.core import XBlock
from xblock.fragment import Fragment

from audio.utils import load_resource, render_template

class StudioMixin(object):
    """
    Studio editing view for Tagged Text XBlock
    """

    def studio_view(self, context=None):
        """
        Method to render the Tagged Text XBlock in Studio
        """

        data = {
            'metadata_fields': json.dumps(self.editable_metadata_fields)
        }

        template = render_template('templates/studio.html', data)
        frag = Fragment(template)
        frag.add_javascript(load_resource('static/script/xblock-audio.min.js'))
        frag.initialize_js('AudioXBlockStudio')
        return frag

    @XBlock.json_handler
    def edit(self, data, suffix=''):
        """
        Update the XBlock's settings
        """
        editable_fields = self.editable_metadata_fields

        for metadata_key, metadata in data['metadata'].items():
            if metadata_key not in self.fields:
                return {'success': False, 'msg': 'Field "{name}" does not exist'.format(name=metadata_key)}

            if metadata_key not in editable_fields:
                return {'success': False, 'msg': 'Field "{name}" is not editable'.format(name=metadata_key)}

            value = metadata.get('value')
            field = self.fields[metadata_key]

            if value is None:
                field.delete_from(self)
            else:
                try:
                    value = field.from_json(value)
                except ValueError:
                    return {'success': False, 'msg': 'Invalid data for field {name}'.format(name=metadata_key)}

                field.write_to(self, value)

        return {'success': True, 'msg': 'Successfully updated Audio XBlock'}


import inspect

from django.db import models
from django.contrib.auth.models import User

from mess.membership import models as m_models
from mess.events import models as e_models
from mess.scheduling import models as s_models

class RevisionModel(models.Model):
  """
  Base class for all revision models.

  Main reason to create a base class is
  to overload the == operator. When
  compared, revisions should be judged
  equal if their content is equal, not their
  primary key ids (which is the default in
  comparing django models)
  """

  def __eq__(self, other):
    """
    Two revisions are equal if the fields specified
    in the eq_fields property are equal.

    eq_fields are either basic types OR managers
    if the field is a ManyToMany field.
    """

    if type(self) != type(other):
      return False

    for i in range(len(self.eq_fields)):
      if isinstance(self.eq_fields[i], models.manager.Manager):

        s_objects = self.eq_fields[i].order_by('id')
        o_objects = other.eq_fields[i].order_by('id')

        if len(s_objects) != len(o_objects):
          return False

        for s,o in zip(s_objects, o_objects):
          if s.id != o.id:
            return False

      else:
        if self.eq_fields[i] != other.eq_fields[i]:
          return False

    return True

  @property
  def eq_fields(self):
    """
    Defaul eq_fields default to ID, which is the default
    comparison field for django models as a whole
    """
    return [id]

class MemberRevision(RevisionModel):

  member = models.ForeignKey(m_models.Member, related_name="member_revisions")
  user = models.ForeignKey(User)
  status = models.CharField(max_length=1, null=True)
  work_status = models.CharField(max_length=1, null=True)
  has_key = models.BooleanField(default=False)
  date_joined = models.DateField(null=True)
  date_missing = models.DateField(blank=True, null=True)
  date_departed = models.DateField(blank=True, null=True)
  date_turns_18 = models.DateField(blank=True, null=True)
  card_number = models.CharField(max_length=128, blank=True, null=True)
  card_facility_code = models.CharField(max_length=128, blank=True, null=True)
  card_type = models.CharField(max_length=128, blank=True, null=True)
  member_owner_equity_held = models.DecimalField(max_digits=8, decimal_places=2, null=True)
  membership_fund_equity_held = models.DecimalField(max_digits=8, decimal_places=2, null=True)
  equity_due = models.DecimalField(max_digits=8, decimal_places=2, null=True)
  equity_increment = models.DecimalField(max_digits=8, decimal_places=2, null=True)
  referral_source = models.CharField(max_length=20, blank=True, null=True)
  referring_member = models.ForeignKey(m_models.Member, blank=True, null=True)
  orientation = models.ForeignKey(e_models.Orientation, blank=True, null=True)
  job_interests = models.ManyToManyField(s_models.Job)
  skills = models.ManyToManyField(s_models.Skill)
  availability = models.IntegerField(null=True)
  extra_info = models.CharField(max_length=255, blank=True, null=True)
  timestamp = models.DateTimeField(auto_now_add=True)

  @classmethod
  def create_member_revision(cls, member=None):
    member_revision = cls()

    if member:
      member_revision.member = member
      member_revision.user = member.user
      member_revision.status = member.status
      member_revision.work_status = member.work_status
      member_revision.has_key = member.has_key
      member_revision.date_joined = member.date_joined
      member_revision.date_missing = member.date_missing
      member_revision.date_departed = member.date_departed
      member_revision.date_turns_18 = member.date_turns_18
      member_revision.card_number = member.card_number
      member_revision.card_facility_code = member.card_facility_code
      member_revision.card_type = member.card_type
      member_revision.member_owner_equity_held = member.member_owner_equity_held
      member_revision.membership_fund_equity_held = member.membership_fund_equity_held
      member_revision.equity_due = member.equity_due
      member_revision.equity_increment = member.equity_increment
      member_revision.referral_source = member.referral_source
      member_revision.referring_member = member.referring_member
      member_revision.orientation = member.orientation
      member_revision.availability = member.availability
      member_revision.extra_info = member.extra_info

      member_revision.save()

      for job_interest in member.job_interests.all():
        member_revision.job_interests.add(job_interest)

      for skill in member.skills.all():
        member_revision.skills.add(skill)

      member_revision.save()

    return member_revision

  @property
  def eq_fields(self):
    return [
        self.member,
        self.user,
        self.status,
        self.work_status,
        self.has_key,
        self.date_joined,
        self.date_missing,
        self.date_departed,
        self.date_turns_18,
        self.card_number,
        self.card_facility_code,
        self.card_type,
        self.member_owner_equity_held,
        self.membership_fund_equity_held,
        self.equity_due,
        self.equity_increment,
        self.referral_source,
        self.referring_member,
        self.orientation,
        self.availability,
        self.extra_info,
        self.job_interests,
        self.skills,
        ]

class AccountRevision(RevisionModel):
  account = models.ForeignKey(m_models.Account, related_name="account_revisions")
  name = models.CharField(max_length=50)
  members = models.ManyToManyField(m_models.Member, related_name='historical_accounts')
  can_shop = models.BooleanField()
  ebt_only = models.BooleanField()
  hours_balance = models.DecimalField(max_digits=5, decimal_places=2)
  deposit = models.DecimalField(max_digits=8, decimal_places=2)
  balance_limit = models.DecimalField(max_digits=8, decimal_places=2)
  note = models.TextField(blank=True)
  shared_address = models.BooleanField()
  timestamp = models.DateTimeField(auto_now_add=True)

  @classmethod
  def create_account_revision(cls, account=None):
    account_revision = cls()

    if account:
      account_revision.account = account
      account_revision.name = account.name
      account_revision.can_shop = account.can_shop
      account_revision.ebt_only = account.ebt_only
      account_revision.hours_balance = account.hours_balance
      account_revision.deposit = account.deposit
      account_revision.balance_limit = account.balance_limit
      account_revision.note = account.note
      account_revision.shared_address = account.shared_address

      account_revision.save()

      for member in account.members.all():
        account_revision.members.add(member)

      account_revision.save()

    return account_revision

  @property
  def eq_fields(self):
    return [
      self.account,
      self.name,
      self.members,
      self.can_shop,
      self.ebt_only,
      self.hours_balance,
      self.deposit,
      self.balance_limit,
      self.note,
      self.shared_address,
      ]


import psycopg2
import argparse
from flask import Flask, request, Response, jsonify, make_response
import itertools, traceback
import json, pandas
import time
from typing import Optional, List
from io import BytesIO

_version = '2021-01-30'

class Properties:
    def __init__(self, db_addr: str, db_port: int, db_name: str, db_user: str, db_pass: str, api_port: int):
        self.db_addr = db_addr
        self.db_port = db_port
        self.db_name = db_name
        self.db_user = db_user
        self.db_pass = db_pass
        self.api_port = api_port
        self._conn: Optional[psycopg2.extensions.connection] = None
    @property
    def conn_string(self) -> str:
        return f'host={self.db_addr} port={self.db_port} dbname={self.db_name}' \
                f' user={self.db_user} password={self.db_pass}'
    @property
    def conn(self):
        if self._conn is None or self._conn.closed:
            self._conn = psycopg2.connect(self.conn_string)
        return self._conn
    def close(self):
        if self._conn is not None:
            if not self._conn.closed:
                self._conn.close()
            self._conn = None

app = Flask(__name__)
props: Properties

@app.after_request
def after_request(response) -> Response:
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = '*'
    response.headers['Access-Control-Allow-Headers'] = '*'
    return response

def drop_tables() -> None:
    tables = ('messages', 'users_groups', 'operations', 'user_group_statuses', 'operation_types', 'groups', 'users')
    with props.conn.cursor() as cur:
        for table in tables:
            cur.execute(f'DROP table {table}')
        props.conn.commit()

def ensure_tables() -> None:
    with open('database_init.sql') as f:
        init: List[str] = f.read().split(';')
    with props.conn.cursor() as cur:
        for statement in init:
            if statement.strip() != '':
                cur.execute(statement)

        needed_statuses = ('creator', 'admin', 'user', 'pending', 'blocked')
        cur.execute('SELECT name FROM user_group_statuses')
        statuses = list(map(lambda x: x[0], cur.fetchall()))
        for status in needed_statuses:
            if status not in statuses:
                cur.execute('INSERT INTO user_group_statuses (name) VALUES (%s)', (status,))

        needed_operation_types = ('income', 'spending')
        cur.execute('SELECT name FROM operation_types')
        operation_types = list(map(lambda x: x[0], cur.fetchall()))
        for op_type in needed_operation_types:
            if op_type not in operation_types:
                cur.execute('INSERT INTO operation_types (name) VALUES (%s)', (op_type,))
        props.conn.commit()

# groups

@app.route('/group/<int:group_id>/', methods = ['GET'])
def get_group(group_id: int) -> Response:
    status: Optional[str] = request.args.get('status')
    with props.conn.cursor() as cur:
        cur.execute('SELECT g.name, g.creator_id, u.username, g.balance FROM groups g'
                '   JOIN users u ON u.id = g.creator_id WHERE g.id = %s', (group_id,))
        res = cur.fetchone()
        if res is None:
            return make_response(jsonify({'error': f'group with id={group_id} is not found'}), 404)
        name, creator_id, creator, balance = res
        cur.execute('SELECT u.id, u.username, ugs.name FROM users u'
                '   JOIN users_groups ug ON u.id = ug.user_id'
                '   JOIN groups g ON g.id = ug.group_id'
                '   JOIN user_group_statuses ugs on ug.status_id = ugs.id'
                '   WHERE g.id = %s' + ('' if status is None else ' AND ugs.name = %s'),
                (group_id,) if status is None else (group_id, status))
        res = pandas.DataFrame(cur.fetchall(), columns = ('id', 'username', 'status'))
        return make_response(jsonify({
            'group': {
                'id': group_id,
                'name': name,
                'creator_id': creator_id,
                'creator': creator,
                'balance': balance,
                'users': list(res.transpose().to_dict().values())
            }
        }))

@app.route('/group/', methods = ['POST'])
def add_group() -> Response:
    body = json.loads(request.data)
    if not ('name' in body and 'user' in body):
        raise Exception('Missing one of the (name, user) in request body')
    id = body['user'].isnumeric()
    with props.conn.cursor() as cur:
        cur.execute('INSERT INTO groups (name, creator_id) VALUES (%s, '
                + ('%s' if id else '(SELECT id FROM users WHERE username = %s)')
                + ') RETURNING id',
                (body['name'], body['user']))
        id = cur.fetchone()[0]
        cur.execute('INSERT INTO users_groups (user_id, group_id, status_id) VALUES'
                '   ((SELECT id FROM users where username = %s), %s, (SELECT id from user_group_statuses where name = \'creator\')) RETURNING id',
                (body['user'], id))
        props.conn.commit()
        return make_response(jsonify({'group_id': id, 'user_group_id': cur.fetchone()[0]}))

@app.route('/group/<int:group_id>/', methods = ['DELETE'])
def delete_group(group_id) -> Response:
    if not 'user' in request.args:
        return make_response(jsonify({'error': f'user parameter is missing in request path'}), 400)
    id = request.args['user'].isnumeric()
    with props.conn.cursor() as cur:
        cur.execute('SELECT ugs.name FROM users_groups ug JOIN user_group_statuses ugs on ug.status_id = ugs.id ' +
                ('WHERE ug.user_id = %s' if id else 'JOIN users u on ug.user_id = u.id WHERE u.name = %s'), (request.args['user'],))
        res = cur.fetchone()
        if res is None or res[0] != 'creator':
            return make_response(jsonify({'error': 'user does not have enough right to delete group'}))
        cur.execute('DELETE FROM groups WHERE id = %s', (id,))
        props.conn.commit()
        return make_response(jsonify({'result': f'deleted group with id={id}'}))

# users - groups

@app.route('/user/<user>/groups/', methods = ['GET'])
def get_user_groups(user: str) -> Response:
    id = user.isnumeric()
    status: Optional[str] = request.args.get('status')
    with props.conn.cursor() as cur:
        cur.execute('SELECT g.id, g.name, uc.count, ugs.name, g.creator_id, u.username, g.balance FROM groups g'
                '   JOIN users_groups ug ON g.id = ug.group_id'
                '   JOIN (SELECT group_id, count(*) FROM users_groups WHERE '
                '       status_id in (select id from user_group_statuses where name = \'user\' or name = \'admin\' or name = \'creator\') GROUP BY group_id)'
                '           as uc ON g.id = uc.group_id'
                '   JOIN users u ON u.id = g.creator_id'
                '   JOIN user_group_statuses ugs on ug.status_id = ugs.id' +
                ('  WHERE ug.user_id = %s' if id else ' WHERE ug.user_id = (SELECT id FROM users WHERE username = %s)') +
                ('  AND ug.status_id = ugs.name = %s' if status is not None else '') +
                '   ORDER BY g.id', (user, status) if status is not None else (user,))
        res = pandas.DataFrame(cur.fetchall(), columns = ('id', 'name', 'size', 'status', 'creator_id', 'creator', 'balance'))
        return make_response(jsonify({
            'groups': list(res.transpose().to_dict().values())
        }))

@app.route('/group/<int:group_id>/join/', methods = ['POST'])
def user_to_group(group_id: int) -> Response:
    if not 'user' in request.args:
        return make_response(jsonify({'error': f'user parameter is missing in request path'}), 400)
    id = request.args['user'].isnumeric()

    with props.conn.cursor() as cur:
        cur.execute('INSERT INTO users_groups (user_id, group_id, status_id) VALUES ('
                + ('%s' if id else'(SELECT id FROM users WHERE username = %s)')
                + ', %s, (SELECT id FROM user_group_statuses WHERE name = \'pending\')) ON CONFLICT DO NOTHING',
                (request.args['user'], group_id))
        props.conn.commit()
        return make_response(jsonify({'result': 'ok'}))

@app.route('/group/<int:group_id>/status/<user>/', methods = ['PUT'])
def user_set_status(group_id: int, user: str) -> Response:
    if not 'user' in request.args:
        return make_response(jsonify({'error': f'user parameter is missing in request path'}), 400)
    if not 'status' in request.args:
        return make_response(jsonify({'error': 'new status is missing in request path'}), 400)
    user_id = user.isnumeric()
    requester_id = request.args['user'].isnumeric()
    with props.conn.cursor() as cur:
        cur.execute('SELECT id FROM user_group_statuses WHERE name = %s', (request.args['status'],))
        status_id = cur.fetchone()
        if status_id is None:
            return make_response(jsonify({'error': 'status is not found'}))
        status_id = status_id[0]
        cur.execute('SELECT s.name FROM users_groups ug JOIN user_group_statuses s on ug.status_id = s.id WHERE ug.group_id = %s AND ug.user_id = '
                + ('%s' if requester_id else '(SELECT id FROM users WHERE username = %s)'),
                (group_id, request.args['user']))
        res = cur.fetchone()
        if res is None or res[0] not in ('admin', 'creator') or request.args['status'] == 'creator' or (res[0] == 'admin' and request.args['status'] == 'admin'):
            return make_response(jsonify({'error': 'not enough rights to change someone else\'s status'}), 403)
        cur.execute('SELECT id FROM users_groups WHERE group_id = %s AND user_id = '
                + ('%s' if id else '(SELECT id FROM users WHERE username = %s)'),
                (group_id, user))
        res = cur.fetchone()
        if res is None:
            return make_response(jsonify({'error': f'user ({user}) is not found in the group ({group_id})'}), 400)
        ug_id = res[0]

        cur.execute('UPDATE users_groups SET status_id = %s WHERE id = %s', (status_id, ug_id))
        props.conn.commit()
        return make_response(jsonify({'result': 'ok'}))

# users

@app.route('/user/', methods = ['POST'])
def add_user() -> Response:
    body = json.loads(request.data)
    if not ('username' in body and 'password' in body):
        raise Exception('Missing one of the (username, password) in request body')
    if len(body['username']) < 2:
        return make_response(jsonify({'error': 'username must be at least 2 characters long'}), 400)
    if body['username'].isnumeric():
        return make_response(jsonify({'error': 'username must contain at least one letter or other symbol'}), 400)
    if sum(filter(lambda sym: sym in body['username'], list('/& '))) != 0:
        return make_response(jsonify({'error': 'username contains illegal characters'}))
    try:
        with props.conn.cursor() as cur:
            t = time.localtime()
            cur.execute('INSERT INTO users (username, password, registration_date) VALUES (%s, %s, %s) RETURNING id',
                    (body['username'], body['password'], f'{t.tm_year}-{t.tm_mon}-{t.tm_mday} {t.tm_hour}:{t.tm_min}:{t.tm_sec}'))
            props.conn.commit()
            return make_response(jsonify({'result': f'added user with id={cur.fetchone()[0]}'}))
    except psycopg2.DatabaseError as ex:
        print(ex)
        props.conn.rollback()
        return make_response(jsonify({'error': f"User with username '{body['username']} already exists"}))

@app.route('/user/<int:id>/', methods = ['GET'])
def get_user(id: int) -> Response:
    with props.conn.cursor() as cur:
        cur.execute('SELECT username FROM users WHERE id = %s', (id,))
        res = cur.fetchall()
        if len(res) == 0:
            return make_response(jsonify({'error': 'user not found'}), 404)
        return make_response(jsonify({'username': res[0]}))

@app.route('/login/', methods = ['POST'])
def login() -> Response:
    body = json.loads(request.data)
    if not ('username' in body and 'password' in body):
        return make_response(jsonify({'error': 'request body is missing username or password fields'}), 400)
    with props.conn.cursor() as cur:
        cur.execute('SELECT password FROM users WHERE username = %s', (body['username'],))
        res = cur.fetchall()
        if len(res) == 0 or res[0][0] != body['password']:
            return make_response(jsonify({'result': 'wrong username or password'}), 403)
        return make_response(jsonify({'result': 'ok'}))

# operations

@app.route('/group/<int:group_id>/operations/', methods = ['GET'])
def get_operations(group_id: int) -> Response:
    if not 'user' in request.args:
        return make_response(jsonify({'error': f'user parameter is missing in request path'}), 400)
    id = request.args['user'].isnumeric()
    page = request.args.get('page', 0)
    with props.conn.cursor() as cur:
        cur.execute('SELECT id FROM users_groups WHERE group_id = %s AND user_id = '
                + ('%s' if id else '(SELECT id FROM users WHERE username = %s)'),
                (group_id, request.args['user']))
        res = cur.fetchone()
        if res is None:
            return make_response(jsonify({'error': f'user ({request.args["user"]}) is not found in the group ({group_id})'}), 403)
        ug_id = res[0]
        cur.execute('SELECT o.id, u.username, ot.name, o.amount, o.name, o.description, o.date FROM operations o JOIN users u ON o.user_id = u.id'
                ' JOIN operation_types ot ON o.type_id = ot.id WHERE o.group_id = %s LIMIT %s OFFSET %s', (group_id, 50, page * 50))
        res = pandas.DataFrame(cur.fetchall(), columns=('id', 'user', 'type', 'amount', 'name', 'description', 'date'))
        res['date'] = res['date'].apply(lambda t: f'{t.year:02}-{t.month:02}-{t.day:02} {t.hour:02}:{t.minute:02}:{t.second:02}')
        return make_response(jsonify({
            'operations': list(res.transpose().to_dict().values())
        }))

@app.route('/group/<int:group_id>/operation/', methods = ['POST'])
def create_operation(group_id):
    body = json.loads(request.data)
    if not ('user' in body and 'type' in body and 'amount' in body and 'name' in body):
        return make_response(jsonify({'error': 'request body is missing user, type, amount or name fields'}), 400)
    if body['type'] not in ('income', 'spending'):
        return make_response(jsonify({'error': f"operation type must be one of the ('income', 'spending'), but not '{body['type']}'"}), 400)
    try:
        amount = float(body['amount'])
    except ValueError:
        return make_response(jsonify({'error': f'amount must be a floating point number (but is {body["amount"]})'}))
    id = isinstance(body['user'], int)
    with props.conn.cursor() as cur:
        cur.execute('SELECT id FROM users_groups WHERE group_id = %s AND user_id = '
                + ('%s' if id else '(SELECT id FROM users WHERE username = %s)'),
                (group_id, body['user']))
        res = cur.fetchone()
        if res is None:
            return make_response(jsonify({'error': f'user ({body["user"]}) is not found in the group ({group_id})'}), 400)
        ug_id = res[0]
        t = time.localtime()
        cur.execute('INSERT INTO operations (user_id, group_id, type_id, amount, name, description, date) VALUES ('
                + ('%s' if id else '(SELECT id FROM users WHERE username = %s)')
                + ', %s, (SELECT id FROM operation_types WHERE name = %s), %s, %s, %s, %s)',
                (body['user'], group_id, body['type'], amount, body['name'], body.get('description', ''),
                    f'{t.tm_year}-{t.tm_mon}-{t.tm_mday} {t.tm_hour}:{t.tm_min}:{t.tm_sec}'))
        cur.execute('UPDATE groups SET balance = balance ' + ('+' if body['type'] == 'income' else '-') + ' %s WHERE id = %s', (amount, group_id))
        props.conn.commit()
        return make_response(jsonify({'result': 'ok'}))

# chats

@app.route('/group/<int:group_id>/chat/', methods = ['GET'])
def get_chat(group_id: int):
    if not 'user' in request.args:
        return make_response(jsonify({'error': f'user parameter is missing in request path'}), 400)
    id = request.args['user'].isnumeric()
    page = int(request.args.get('page', 0))
    with props.conn.cursor() as cur:
        cur.execute('SELECT count(*) FROM users_groups '
                + ('WHERE group_id = %s AND user_id = %s' if id else
                        'ug JOIN users u ON u.id = ug.user_id WHERE ug.group_id = %s AND u.username = %s'), (group_id, request.args['user'],))
        if cur.fetchone()[0] != 1:
            return make_response(jsonify({'error': f'user ({request.args["user"]}) is not in the given group ({group_id})'}), 403)
        cur.execute('SELECT m.id, ug.user_id, u.username, m.message, m.time FROM users_groups ug JOIN users u ON ug.user_id = u.id'
                '   JOIN messages m on m.user_group_id = ug.id WHERE ug.group_id = %s ORDER BY id DESC LIMIT %s OFFSET %s', (group_id, 50, 50 * page))
        res = pandas.DataFrame(cur.fetchall(), columns=('message_id', 'user_id', 'user', 'message', 'time'))
        res['time'] = res['time'].apply(lambda t: f'{t.year:02}-{t.month:02}-{t.day:02} {t.hour:02}:{t.minute:02}:{t.second:02}')
        return make_response(jsonify({
            'messages': list(res.transpose().to_dict().values())
        }))

@app.route('/group/<int:group_id>/chat/', methods = ['POST'])
def send_to_chat(group_id: int):
    body = json.loads(request.data)
    if not 'user' in body:
        return make_response(jsonify({'error': f'user parameter is missing in request body'}))
    id = body['user'].isnumeric()
    with props.conn.cursor() as cur:
        cur.execute('SELECT ug.id FROM users_groups ug JOIN users u ON u.id = ug.user_id WHERE u.username = %s' if id else 'WHERE ug.id = %s', (body['user'],))
        res = cur.fetchall()
        if len(res) == 0:
            return make_response(jsonify({'error': f'user ({body["user"]}) is not in the given group ({group_id})'}), 403)
        cur.execute('INSERT INTO messages (user_group_id, time, message) VALUES (%s, %s %s)', (res[0][0]))
        props.conn.commit()
        return make_response(jsonify({'result': 'ok'}))

# api help

@app.route('/', methods = ['GET'])
@app.route('/api/', methods = ['GET'])
def api():
    return make_response(jsonify({
        'version': _version,
        '_links': {
            'self': {
                'href': '/api/'
            }
        }
    }))

# errors handling

@app.errorhandler(Exception)
def any_error(error: Exception) -> Response:
    props.conn.rollback()
    print(f'path: {request.path}?{"&".join(map(lambda x: f"{x[0]}={x[1]}", request.args.items()))}, body={request.data.decode()}')
    traceback.print_exc()
    return make_response(jsonify({
        'error': str(error),
        'error_type': str(type(error)),
        'path': request.path,
        'body': request.data.decode(),
        'params': '&'.join(map(lambda x: f'{x[0]}={x[1]}', request.args.items())),
        'trace': list(itertools.chain(*map(lambda x: x.split('\n'), traceback.format_tb(error.__traceback__))))
    }), 500)

@app.errorhandler(404)
def not_found_error(_) -> Response:
    return make_response(jsonify({
        'error': 'not found',
        'path': request.path,
        'params': '&'.join(map(lambda x: f'{x[0]}={x[1]}', request.args.items())),
    }), 404)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Starts up the finances app API server')
    parser.add_argument('-H', '--db_addr', action='store', dest='db_addr',
                        help=f'postgres host address', type=str, default='localhost')
    parser.add_argument('-P', '--db_port', action='store', dest='db_port',
                        help=f'postgres port number', type=int, default=5432)
    parser.add_argument('-d', '--db_name', action='store', dest='db_name',
                        help=f'postgres database name', type=str, default='finances')
    parser.add_argument('-U', '--db_user', action='store', dest='db_user',
                        help=f'postgres user name', type=str, default='postgres')
    parser.add_argument('-W', '--db_pass', action='store', dest='db_pass',
                        help=f'database user password', type=str, default='postgres')
    parser.add_argument('-p', '--port', action='store', dest='api_port',
                        help=f'api port number', type=int, default=3001)
    args = parser.parse_args()

    props = Properties(args.db_addr, args.db_port, args.db_name, args.db_user, args.db_pass, args.api_port)

    print(f'Starting finances app API server at port {props.api_port}.')
    print(f'Using postgresql database: {props.db_user}@{props.db_addr}:{props.db_port}/{props.db_name}')

    ensure_tables()
    app.run(host='0.0.0.0', port=props.api_port)


from tkinter import *
from tkinter import filedialog as fd
import tkinter as tk
import webbrowser as wb
import os

class ParentWindow(Frame):

    def __init__(self, master):
        Frame.__init__(self) # initialize frame

        self.filename = StringVar()
        self.fileText = StringVar()

        def createPage(self):
            self.filename = self.filename.get() + ".html" # append .html file extension to filename

            if os.path.exists("{}".format(self.filename)): # filename already exists, clear file
                os.remove("{}".format(self.filename))
            else:
                pass
            file = open("{}".format(self.filename), "a") # create new file
            # write text extracted from textbox (fileText) to filename with HTML appended
            file.write("\n\n
\n{}\n
\n\n".format(self.fileText.get()))
            file.close() # close file
            file = open("{}".format(self.filename), "r") # open file as read only
            wb.open_new_tab(self.filename) # open file in a tab in web browser

        self.master = master
        self.master.resizable(width=False, height=False)
        self.master.geometry('{}x{}'.format(560, 190))
        self.master.title('Create WebPage')

        # labels
        self.lblFilename = tk.Label(self.master, text='Enter Filename: ')
        self.lblFilename.grid(row=0, column=0, padx=(30, 20), pady=(10, 10), sticky=N+W)
        self.lblFileText = tk.Label(self.master, text='Enter Webpage Text: ')
        self.lblFileText.grid(row=1, column=0, padx=(30, 20), pady=(10, 10), sticky=N+W)

        # textboxes
        self.txtFilename = tk.Entry(self.master, width=50, text=self.filename)
        self.txtFilename.grid(row=0, column=1, columnspan=4, padx=(30, 20), pady=(10, 10), sticky=N+E+W)
        self.txtFileText = tk.Entry(self.master, width=50,text=self.fileText)
        self.txtFileText.grid(row=1, column=1, columnspan=4, padx=(30, 20), pady=(10, 10), sticky=N+E+W)

        #buttons
        self.btn_submit = tk.Button(self.master, width=12, height=2, text='Submit', command=lambda: createPage(self))
        self.btn_submit.grid(row=2, column=0, padx=(30, 20), pady=(10, 10), sticky=N+E+W)

# main driver
if __name__ == "__main__":
    root = tk.Tk()
    App = ParentWindow(root)
    root.mainloop()


# Ranger deep learning optimizer - RAdam + Lookahead + Gradient Centralization, combined into one optimizer.

# https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer
# and/or
# https://github.com/lessw2020/Best-Deep-Learning-Optimizers

# Ranger has now been used to capture 12 records on the FastAI leaderboard.

# This version = 20.4.11

# Credits:
# Gradient Centralization --> https://arxiv.org/abs/2004.01461v2 (a new optimization technique for DNNs), github:  https://github.com/Yonghongwei/Gradient-Centralization
# RAdam -->  https://github.com/LiyuanLucasLiu/RAdam
# Lookahead --> rewritten by lessw2020, but big thanks to Github @LonePatient and @RWightman for ideas from their code.
# Lookahead paper --> MZhang,G Hinton  https://arxiv.org/abs/1907.08610

# summary of changes:
# 4/11/20 - add gradient centralization option.  Set new testing benchmark for accuracy with it, toggle with use_gc flag at init.
# full code integration with all updates at param level instead of group, moves slow weights into state dict (from generic weights),
# supports group learning rates (thanks @SHolderbach), fixes sporadic load from saved model issues.
# changes 8/31/19 - fix references to *self*.N_sma_threshold;
# changed eps to 1e-5 as better default than 1e-8.

import math

import torch
from torch.optim.optimizer import Optimizer, required

class Ranger(Optimizer):
    def __init__(
        self,
        params,
        lr=1e-3,  # lr
        alpha=0.5,
        k=6,
        N_sma_threshhold=5,  # Ranger options
        betas=(0.95, 0.999),
        eps=1e-5,
        weight_decay=0,  # Adam options
        # Gradient centralization on or off, applied to conv layers only or conv + fc layers
        use_gc=True,
        gc_conv_only=False,
    ):

        # parameter checks
        if not 0.0 <= alpha <= 1.0:
            raise ValueError(f"Invalid slow update rate: {alpha}")
        if not 1 <= k:
            raise ValueError(f"Invalid lookahead steps: {k}")
        if not lr > 0:
            raise ValueError(f"Invalid Learning Rate: {lr}")
        if not eps > 0:
            raise ValueError(f"Invalid eps: {eps}")

        # parameter comments:
        # beta1 (momentum) of .95 seems to work better than .90...
        # N_sma_threshold of 5 seems better in testing than 4.
        # In both cases, worth testing on your dataset (.90 vs .95, 4 vs 5) to make sure which works best for you.

        # prep defaults and init torch.optim base
        defaults = dict(
            lr=lr,
            alpha=alpha,
            k=k,
            step_counter=0,
            betas=betas,
            N_sma_threshhold=N_sma_threshhold,
            eps=eps,
            weight_decay=weight_decay,
        )
        super().__init__(params, defaults)

        # adjustable threshold
        self.N_sma_threshhold = N_sma_threshhold

        # look ahead params

        self.alpha = alpha
        self.k = k

        # radam buffer for state
        self.radam_buffer = [[None, None, None] for ind in range(10)]

        # gc on or off
        self.use_gc = use_gc

        # level of gradient centralization
        self.gc_gradient_threshold = 3 if gc_conv_only else 1

        # print(f"Ranger optimizer loaded. \nGradient Centralization usage = {self.use_gc}")
        if self.use_gc and self.gc_gradient_threshold == 1:
            print(f"GC applied to both conv and fc layers")
        elif self.use_gc and self.gc_gradient_threshold == 3:
            print(f"GC applied to conv layers only")

    def __setstate__(self, state):
        print("set state called")
        super(Ranger, self).__setstate__(state)

    def step(self, closure=None):
        loss = None
        # note - below is commented out b/c I have other work that passes back the loss as a float, and thus not a callable closure.
        # Uncomment if you need to use the actual closure...

        # if closure is not None:
        # loss = closure()

        # Evaluate averages and grad, update param tensors
        for group in self.param_groups:

            for p in group["params"]:
                if p.grad is None:
                    continue
                grad = p.grad.data.float()

                if grad.is_sparse:
                    raise RuntimeError(
                        "Ranger optimizer does not support sparse gradients"
                    )

                p_data_fp32 = p.data.float()

                state = self.state[p]  # get state dict for this param

                if (
                    len(state) == 0
                ):  # if first time to run...init dictionary with our desired entries
                    # if self.first_run_check==0:
                    # self.first_run_check=1
                    # print("Initializing slow buffer...should not see this at load from saved model!")
                    state["step"] = 0
                    state["exp_avg"] = torch.zeros_like(p_data_fp32)
                    state["exp_avg_sq"] = torch.zeros_like(p_data_fp32)

                    # look ahead weight storage now in state dict
                    state["slow_buffer"] = torch.empty_like(p.data)
                    state["slow_buffer"].copy_(p.data)

                else:
                    state["exp_avg"] = state["exp_avg"].type_as(p_data_fp32)
                    state["exp_avg_sq"] = state["exp_avg_sq"].type_as(p_data_fp32)

                # begin computations
                exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"]
                beta1, beta2 = group["betas"]

                # GC operation for Conv layers and FC layers
                if grad.dim() > self.gc_gradient_threshold:
                    grad.add_(-grad.mean(dim=tuple(range(1, grad.dim())), keepdim=True))

                state["step"] += 1

                # compute variance mov avg
                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
                # compute mean moving avg
                exp_avg.mul_(beta1).add_(1 - beta1, grad)

                buffered = self.radam_buffer[int(state["step"] % 10)]

                if state["step"] == buffered[0]:
                    N_sma, step_size = buffered[1], buffered[2]
                else:
                    buffered[0] = state["step"]
                    beta2_t = beta2 ** state["step"]
                    N_sma_max = 2 / (1 - beta2) - 1
                    N_sma = N_sma_max - 2 * state["step"] * beta2_t / (1 - beta2_t)
                    buffered[1] = N_sma
                    if N_sma > self.N_sma_threshhold:
                        step_size = math.sqrt(
                            (1 - beta2_t)
                            * (N_sma - 4)
                            / (N_sma_max - 4)
                            * (N_sma - 2)
                            / N_sma
                            * N_sma_max
                            / (N_sma_max - 2)
                        ) / (1 - beta1 ** state["step"])
                    else:
                        step_size = 1.0 / (1 - beta1 ** state["step"])
                    buffered[2] = step_size

                if group["weight_decay"] != 0:
                    p_data_fp32.add_(-group["weight_decay"] * group["lr"], p_data_fp32)

                # apply lr
                if N_sma > self.N_sma_threshhold:
                    denom = exp_avg_sq.sqrt().add_(group["eps"])
                    p_data_fp32.addcdiv_(-step_size * group["lr"], exp_avg, denom)
                else:
                    p_data_fp32.add_(-step_size * group["lr"], exp_avg)

                p.data.copy_(p_data_fp32)

                # integrated look ahead...
                # we do it at the param level instead of group level
                if state["step"] % group["k"] == 0:
                    # get access to slow param tensor
                    slow_p = state["slow_buffer"]
                    # (fast weights - slow weights) * alpha
                    slow_p.add_(self.alpha, p.data - slow_p)
                    # copy interpolated weights to RAdam param tensor
                    p.data.copy_(slow_p)

        return loss

class RAdam(Optimizer):
    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
        self.buffer = [[None, None, None] for ind in range(10)]
        super(RAdam, self).__init__(params, defaults)

    def __setstate__(self, state):
        super(RAdam, self).__setstate__(state)

    def step(self, closure=None):

        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:

            for p in group["params"]:
                if p.grad is None:
                    continue
                grad = p.grad.data.float()
                if grad.is_sparse:
                    raise RuntimeError("RAdam does not support sparse gradients")

                p_data_fp32 = p.data.float()

                state = self.state[p]

                if len(state) == 0:
                    state["step"] = 0
                    state["exp_avg"] = torch.zeros_like(p_data_fp32)
                    state["exp_avg_sq"] = torch.zeros_like(p_data_fp32)
                else:
                    state["exp_avg"] = state["exp_avg"].type_as(p_data_fp32)
                    state["exp_avg_sq"] = state["exp_avg_sq"].type_as(p_data_fp32)

                exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"]
                beta1, beta2 = group["betas"]

                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
                exp_avg.mul_(beta1).add_(1 - beta1, grad)

                state["step"] += 1
                buffered = self.buffer[int(state["step"] % 10)]
                if state["step"] == buffered[0]:
                    N_sma, step_size = buffered[1], buffered[2]
                else:
                    buffered[0] = state["step"]
                    beta2_t = beta2 ** state["step"]
                    N_sma_max = 2 / (1 - beta2) - 1
                    N_sma = N_sma_max - 2 * state["step"] * beta2_t / (1 - beta2_t)
                    buffered[1] = N_sma

                    # more conservative since it's an approximated value
                    if N_sma >= 5:
                        step_size = (
                            group["lr"]
                            * math.sqrt(
                                (1 - beta2_t)
                                * (N_sma - 4)
                                / (N_sma_max - 4)
                                * (N_sma - 2)
                                / N_sma
                                * N_sma_max
                                / (N_sma_max - 2)
                            )
                            / (1 - beta1 ** state["step"])
                        )
                    else:
                        step_size = group["lr"] / (1 - beta1 ** state["step"])
                    buffered[2] = step_size

                if group["weight_decay"] != 0:
                    p_data_fp32.add_(-group["weight_decay"] * group["lr"], p_data_fp32)

                # more conservative since it's an approximated value
                if N_sma >= 5:
                    denom = exp_avg_sq.sqrt().add_(group["eps"])
                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
                else:
                    p_data_fp32.add_(-step_size, exp_avg)

                p.data.copy_(p_data_fp32)

        return loss

class RMSpropTF(Optimizer):
    """Implements RMSprop algorithm (TensorFlow style epsilon)

    NOTE: This is a direct cut-and-paste of PyTorch RMSprop with eps applied before sqrt
    to closer match Tensorflow for matching hyper-params.

    Proposed by G. Hinton in his
    `course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.

    The centered version first appears in `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        momentum (float, optional): momentum factor (default: 0)
        alpha (float, optional): smoothing (decay) constant (default: 0.9)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-10)
        centered (bool, optional) : if ``True``, compute the centered RMSProp,
            the gradient is normalized by an estimation of its variance
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_decay (bool, optional): decoupled weight decay as per https://arxiv.org/abs/1711.05101
        lr_in_momentum (bool, optional): learning rate scaling is included in the momentum buffer
            update as per defaults in Tensorflow

    """

    def __init__(
        self,
        params,
        lr=1e-2,
        alpha=0.9,
        eps=1e-10,
        weight_decay=0,
        momentum=0.0,
        centered=False,
        decoupled_decay=False,
        lr_in_momentum=True,
    ):
        if not 0.0 <= lr:
            raise ValueError("Invalid learning rate: {}".format(lr))
        if not 0.0 <= eps:
            raise ValueError("Invalid epsilon value: {}".format(eps))
        if not 0.0 <= momentum:
            raise ValueError("Invalid momentum value: {}".format(momentum))
        if not 0.0 <= weight_decay:
            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
        if not 0.0 <= alpha:
            raise ValueError("Invalid alpha value: {}".format(alpha))

        defaults = dict(
            lr=lr,
            momentum=momentum,
            alpha=alpha,
            eps=eps,
            centered=centered,
            weight_decay=weight_decay,
            decoupled_decay=decoupled_decay,
            lr_in_momentum=lr_in_momentum,
        )
        super(RMSpropTF, self).__init__(params, defaults)

    def __setstate__(self, state):
        super(RMSpropTF, self).__setstate__(state)
        for group in self.param_groups:
            group.setdefault("momentum", 0)
            group.setdefault("centered", False)

    def step(self, closure=None):
        """Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group["params"]:
                if p.grad is None:
                    continue
                grad = p.grad.data
                if grad.is_sparse:
                    raise RuntimeError("RMSprop does not support sparse gradients")
                state = self.state[p]

                # State initialization
                if len(state) == 0:
                    state["step"] = 0
                    state["square_avg"] = torch.ones_like(
                        p.data
                    )  # PyTorch inits to zero
                    if group["momentum"] > 0:
                        state["momentum_buffer"] = torch.zeros_like(p.data)
                    if group["centered"]:
                        state["grad_avg"] = torch.zeros_like(p.data)

                square_avg = state["square_avg"]
                one_minus_alpha = 1.0 - group["alpha"]

                state["step"] += 1

                if group["weight_decay"] != 0:
                    if "decoupled_decay" in group and group["decoupled_decay"]:
                        p.data.add_(-group["weight_decay"], p.data)
                    else:
                        grad = grad.add(group["weight_decay"], p.data)

                # Tensorflow order of ops for updating squared avg
                square_avg.add_(one_minus_alpha, grad.pow(2) - square_avg)
                # square_avg.mul_(alpha).addcmul_(1 - alpha, grad, grad)  # PyTorch original

                if group["centered"]:
                    grad_avg = state["grad_avg"]
                    grad_avg.add_(one_minus_alpha, grad - grad_avg)
                    # grad_avg.mul_(alpha).add_(1 - alpha, grad)  # PyTorch original
                    avg = (
                        square_avg.addcmul(-1, grad_avg, grad_avg)
                        .add(group["eps"])
                        .sqrt_()
                    )  # eps moved in sqrt
                else:
                    avg = square_avg.add(group["eps"]).sqrt_()  # eps moved in sqrt

                if group["momentum"] > 0:
                    buf = state["momentum_buffer"]
                    # Tensorflow accumulates the LR scaling in the momentum buffer
                    if "lr_in_momentum" in group and group["lr_in_momentum"]:
                        buf.mul_(group["momentum"]).addcdiv_(group["lr"], grad, avg)
                        p.data.add_(-buf)
                    else:
                        # PyTorch scales the param update by LR
                        buf.mul_(group["momentum"]).addcdiv_(grad, avg)
                        p.data.add_(-group["lr"], buf)
                else:
                    p.data.addcdiv_(-group["lr"], grad, avg)

        return loss


from rest_framework.test import APITestCase, APIClient
from rest_framework import status
from django.urls import reverse

client = APIClient()

class GetAllPostsTest(APITestCase):

    def test_post_creation(self):

        url = reverse('get-posts')
        data = {
            'id': 7,
            'url': "/posts/7",
            'title': "New substitution rules",
            'body': "balloteelli is a foolxd",
            'created': "2020-05-15",
            'modified': "2020-05-17",
            'tags': [
                "New",
                "substitution",
                "rules"
            ],
            'userId': 8
        }
        response = client.post(url, data, format='json')
        self.assertEqual(response.status_code, 200)


class Solution:
    def getDistances(self, arr: List[int]) -> List[int]:
        length = len(arr)
        indices = defaultdict(list)
        prefix = [0 for i in range(length)]
        suffix = [0 for i in range(length)]
        ans = [0 for i in range(length)]

        for index, value in enumerate(arr):
            indices[value].append(index)

        for key, lst in indices.items():
            cnt = 1
            n = len(lst)

            for idx in range(1, n):
                temp = (lst[idx] - lst[idx - 1]) * cnt
                prefix[lst[idx]] = prefix[lst[idx - 1]]  + temp
                cnt += 1

            cnt = 1
            for idx in range(n - 2, -1, -1):
                temp = (lst[idx + 1] - lst[idx]) * cnt
                suffix[lst[idx]] =  suffix[lst[idx + 1]] + temp
                cnt += 1

        for i in range(length):
            ans[i] = (prefix[i] + suffix[i])

        return ans


# coding: utf-8
from django.views.generic import TemplateView, ListView, View
from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required, permission_required
from django.views.decorators.csrf import csrf_exempt
from app.models import *
from app.forms import *
import requests
import os, json, re
import httplib2

from apiclient.discovery import build
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.http import HttpResponseBadRequest
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.conf import settings
from oauth2client import xsrfutil
from oauth2client.client import flow_from_clientsecrets
from oauth2client.django_orm import Storage
from bs4 import BeautifulSoup
from flanker import mime
# Create your views here.
CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), '..', 'client_secrets.json')

FLOW = flow_from_clientsecrets(
    CLIENT_SECRETS,
    scope=[
        'https://www.googleapis.com/auth/plus.me',
        'https://mail.google.com/',
        'https://www.googleapis.com/auth/gmail.readonly',
        'https://www.googleapis.com/auth/gmail.modify',
    ],
    redirect_uri='http://lab.mailburn.com/oauth2callback')

class Decode64View(TemplateView):
    template_name = 'decode64.html'

    def post(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        ctx = super(Decode64View, self).get_context_data(**kwargs)
        if self.request.POST and self.request.POST.get('text'):
            s = self.request.POST.get('text').encode('utf-8')
            if self.request.POST.get('urlsafe'):
                ctx['decoded'] = base64.urlsafe_b64decode(s)
            else:
                ctx['decoded'] = base64.b64decode(s)
        return ctx

class HomeView(ListView):
    template_name = 'home.html'
    model = Mail
    queryset = Mail.objects.order_by('-pk')
    context_object_name = 'mails'

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(HomeView, self).dispatch(*args, **kwargs)

    def post(self, request, *args, **kwargs):
        form = MailForm(request.POST)
        if form.is_valid():
            m = form.save()
            m.send()
        return self.get(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        ctx = super(HomeView, self).get_context_data(**kwargs)
        ctx['form'] = MailForm()
        if self.request.user.social_auth.filter(provider='google-oauth2').exists():
            ctx['token'] = self.request.user.social_auth.filter(provider='google-oauth2')[0].extra_data['access_token']
#        if r.status_code == 200:
#            raise KeyError
#            ctx['threads'] = r.json()
        return ctx

@login_required
def auth_return(request):
    if not xsrfutil.validate_token(settings.SECRET_KEY, request.REQUEST['state'],
                                 request.user):
        return  HttpResponseBadRequest()
    credential = FLOW.step2_exchange(request.REQUEST)
    storage = Storage(CredentialsModel, 'id', request.user, 'credential')
    storage.put(credential)
    return HttpResponseRedirect("/")

class ThreadsList(View):
    def get(self, request, *args, **kwargs):
        params = {
            'access_token': request.GET.get('access_token')
        }
        if request.GET.get('maxResults'):
            params['maxResults'] = request.GET.get('maxResults')
        if request.GET.get('pageToken'):
            params['pageToken'] = request.GET.get('pageToken')
        if request.GET.get('q'):
            params['q'] = request.GET.get('q')
        r = requests.get('https://www.googleapis.com/gmail/v1/users/me/threads/',
                        params=params)
        if r.status_code == 200:
            threads = []
            for t in r.json()['threads']:
                thread_snippet = cache.get('thread_{}'.format(t['id']), False)
                nt = t
                if not thread_snippet:
                    rr = requests.get('https://www.googleapis.com/gmail/v1/users/me/threads/{}'.format(t['id']),
                                    params=params)
                    if rr.status_code == 200:
                        ans = rr.json()
                        thread_snippet = ans['messages'][0]['snippet']
                        cache.set('thread_{}'.format(t['id']), thread_snippet)
                        nt['snippet'] = thread_snippet
                else:
                    t['snippet'] = thread_snippet
                threads.append(nt)
            return HttpResponse(json.dumps(threads), content_type='application/json')
        else:
            return HttpResponse(r.text, content_type='application/json', status=r.status_code)

class ThreadsGet(View):
    def get(self, request, *args, **kwargs):
        params = {
            'access_token': request.GET.get('access_token')
        }
        if request.GET.get('maxResults'):
            params['maxResults'] = request.GET.get('maxResults')
        if request.GET.get('pageToken'):
            params['pageToken'] = request.GET.get('pageToken')
        if request.GET.get('q'):
            params['q'] = request.GET.get('q')
        r = requests.get('https://www.googleapis.com/gmail/v1/users/me/threads/{}'.format(kwargs.get('threadId')),
                        params=params)
        if r.status_code == 200:
            ans = r.json()
            msgs = []
            for m in r.json()['messages']:
                mr  = requests.get('https://www.googleapis.com/gmail/v1/users/me/messages/{}'.format(m['id']),
                params={'access_token': request.GET.get('access_token'), 'format': request.GET.get('format')})
                if mr.status_code == 200:
                    ans_msg = {
                        'id': m['id'],
                        'opened': True if cache.get(m['id']) else False,
                        'snippet': mr.json()['snippet']
                    }
                    if request.GET.get('format'):
                        msg_raw = str(mr.json()['raw'])
                        if request.GET.get('decode'):
                            msg = mime.from_string(base64.urlsafe_b64decode(msg_raw))
                            if msg.content_type.is_multipart():
                                for part in msg.parts:
                                    if part.content_type == 'text/plain':
                                        part.body = self.msg_filter(part.body)
                                        break
                                    if part.content_type == 'text/html':
                                        part.body = self.msg_filter(part.body)
                            else:
                                msg.body = self.msg_filter(msg.body)
                            ans_msg['debug'] = mime.python_message_to_string(msg.to_python_message())
                            ans_msg['raw'] = mime.python_message_to_string(msg.to_python_message())
                            ans_msg['raw'] = base64.urlsafe_b64encode(ans_msg['raw'])
                        else:
                            ans_msg['raw'] = msg_raw
                    else:
                        if 'parts' in mr.json()['payload']:
                            self.parse_parts(ans_msg, mr.json()['payload']['parts'])
                        else:
                            self.parse_parts(ans_msg, [mr.json()['payload'],])
                    msgs.append(ans_msg)
            ans['messages'] = msgs
            return HttpResponse(json.dumps(ans), content_type='application/json')
        else:
            return HttpResponse(r.text, content_type='application/json', status=r.status_code)

    def msg_filter(self, msg, remove_tags = False):
        if u'\r\n>' in msg:
            msg = msg.split(u'\r\n>')[0]
        msg = re.split(r'\r\n[-]+\r\n', msg)[0]
        msg = re.split(r'\r\n[-]{2,}', msg)[0]
        msg = re.split(r'\r\n[-]{2,}\s{2,}', msg)[0]
        msg = re.split('\d{1,2} (?u)[\w]+ \d{4} (?u)\w{1}., \d{2}:\d{2}', msg, re.U)[0]
        msg = re.split('\d{2} (?u)[\w]+ \d{4} (?u)\w{1}. \d{2}:\d{2}', msg, re.U)[0]
        msg = re.split(r'\r\n\d{4}-\d{2}-\d{2}', msg)[0]
        msg = re.split(r'On \d{2}.\d{2}.\d{2}, \d{2}:\d{2}', msg)[0]
        msg = re.split(r'\r\n\d{2}.\d{2}.\d{2}, (?u)[\w]+', msg)[0]
        msg = re.split(r'On \d{1,2} [\w]+ \d{4} \d{2}:\d{2}', msg)[0]
        msg = re.split(r'\r\nOn [\w]+, [\w]+ \d{1,2}, \d{4},', msg)[0]
        msg = re.split(r'\r\nOn [\w]+ \d{1,2}, \d{4}, at \d{1,2}:\d{1,2}', msg)[0]
        msg = re.split(r'\r\nOn [\w]+, [\w]+ \d{1,2}, \d{4} at \d{2}:\d{2}', msg)[0]
        msg = re.split(r'\r\n\d{2}.\d{2}.\d{4}, \d{2}:\d{2},', msg)[0]
        msg = re.split(r'\r\n\d{2}.\d{2}.\d{4} \d{2}:\d{2} ', msg)[0]
        msg = re.split(r'\d{2}.\d{2}.\d{4}, \d{2}:\d{2}, \\', msg)[0]
        msg = re.split(r'\r\n\s+From: \w+', msg)[0]
        msg = re.split(r'\r\nBest Regards, \w+ \w+\r\n', msg)[0]
        msg = re.split(r'\n[\w]+, \d{1,2} [\w]+ \d{4} \d{2}:\d{2}', msg)[0]
        msg = re.split(r'\r\n(?u)[\w]+, \d{1,2} (?u)[\w]+ \d{4} (?u)\w{1}.', msg, re.U)[0]
        if u'—\r\nSent from Mailbox' in msg:
            msg = msg.split(u'—\r\nSent from Mailbox')[0]
        if u'—\r\nSent from my iPad' in msg:
            msg = msg.split(u'—\r\nSent from my iPad')[0]
        if u'—\r\nSent from my iPhone' in msg:
            msg = msg.split(u'—\r\nSent from my iPhone')[0]
        if u'\r\nSend from Windows Phone' in msg:
            msg = msg.split(u'\r\nSend from Windows Phone')[0]
        if u'View this email\r\nin your browser' in msg:
            msg = msg.split(u'View this email\r\nin your browser')[0]
        if u'\r\nBest regards' in msg:
            msg = msg.split(u'\r\nBest regards')[0]
        if u'\r\nОтправлено из мобильной Почты Mail.Ru\r\n' in msg:
            msg = msg.split(u'\r\nОтправлено из мобильной Почты Mail.Ru\r\n')[0]
        msg = re.split(r'[\r\n]+$', msg)[0]
        return msg

    def parse_parts(self, msg, parts):
        for part in parts:
            if part['mimeType'] == 'text/plain':
                msg['data'] = base64.urlsafe_b64decode(str(part['body']['data'])).decode('utf-8')
                msg['original_data'] = msg['data']
                if u'\r\n>' in msg['data']:
                    msg['data'] = msg['data'].split(u'\r\n>')[0]
                msg['data'] = re.split(r'\r\n[-]+\r\n', msg['data'])[0]
                msg['data'] = re.split(r'\r\n[-]{2,}', msg['data'])[0]
                msg['data'] = re.split(r'\r\n[-]{2,}\s{2,}', msg['data'])[0]
                msg['data'] = re.split('\d{1,2} (?u)[\w]+ \d{4} (?u)\w{1}., \d{2}:\d{2}', msg['data'], re.U)[0]
                msg['data'] = re.split('\d{2} (?u)[\w]+ \d{4} (?u)\w{1}. \d{2}:\d{2}', msg['data'], re.U)[0]
                msg['data'] = re.split(r'\r\n\d{4}-\d{2}-\d{2}', msg['data'])[0]
                msg['data'] = re.split(r'On \d{2}.\d{2}.\d{2}, \d{2}:\d{2}', msg['data'])[0]
                msg['data'] = re.split(r'\r\n\d{2}.\d{2}.\d{2}, (?u)[\w]+', msg['data'])[0]
                msg['data'] = re.split(r'On \d{1,2} [\w]+ \d{4} \d{2}:\d{2}', msg['data'])[0]
                msg['data'] = re.split(r'\r\nOn [\w]+, [\w]+ \d{1,2}, \d{4},', msg['data'])[0]
                msg['data'] = re.split(r'\r\nOn [\w]+ \d{1,2}, \d{4}, at \d{1,2}:\d{1,2}', msg['data'])[0]
                msg['data'] = re.split(r'\r\nOn [\w]+, [\w]+ \d{1,2}, \d{4} at \d{2}:\d{2}', msg['data'])[0]
                msg['data'] = re.split(r'\r\n\d{2}.\d{2}.\d{4}, \d{2}:\d{2},', msg['data'])[0]
                msg['data'] = re.split(r'\r\n\d{2}.\d{2}.\d{4} \d{2}:\d{2} ', msg['data'])[0]
                msg['data'] = re.split(r'\d{2}.\d{2}.\d{4}, \d{2}:\d{2}, \\', msg['data'])[0]
                msg['data'] = re.split(r'\r\n\s+From: \w+', msg['data'])[0]
                msg['data'] = re.split(r'\r\nBest Regards, \w+ \w+\r\n', msg['data'])[0]
                msg['data'] = re.split(r'\n[\w]+, \d{1,2} [\w]+ \d{4} \d{2}:\d{2}', msg['data'])[0]
                msg['data'] = re.split(r'\r\n(?u)[\w]+, \d{1,2} (?u)[\w]+ \d{4} (?u)\w{1}.', msg['data'], re.U)[0]
                if u'—\r\nSent from Mailbox' in msg['data']:
                    msg['data'] = msg['data'].split(u'—\r\nSent from Mailbox')[0]
                if u'\r\nSend from Windows Phone' in msg['data']:
                    msg['data'] = msg['data'].split(u'\r\nSend from Windows Phone')[0]
                if u'View this email\r\nin your browser' in msg['data']:
                    msg['data'] = msg['data'].split(u'View this email\r\nin your browser')[0]
                if u'\r\nBest regards' in msg['data']:
                    msg['data'] = msg['data'].split(u'\r\nBest regards')[0]
                if u'\r\nОтправлено из мобильной Почты Mail.Ru\r\n' in msg['data']:
                    msg['data'] = msg['data'].split(u'\r\nОтправлено из мобильной Почты Mail.Ru\r\n')[0]
                msg['data'] = re.split(r'[\r\n]+$', msg['data'])[0]
                return
            if part['mimeType'] == 'text/html':
                msg['type'] = 'html'
                msg['data'] = base64.urlsafe_b64decode(str(part['body']['data']))
                soup = BeautifulSoup(msg['data'])
                msg['data'] = u''.join(soup.findAll(text=True))
                msg['data'] = re.split(r'View this conversation on GetMailDone', msg['data'])[0]
                msg['data'] = re.split(r'\d{2}.\d{2}.\d{4}, \d{2}:\d{2}, \"', msg['data'])[0]
            if 'parts' in part:
                self.parse_parts(msg, part['parts'])

import email
import base64
import simplejson
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import time
import hashlib
from django.core.cache import cache
import quopri
import logging
logging.basicConfig(filename='/tmp/debug.log', level=logging.DEBUG)
class MessageSend(View):

    @csrf_exempt
    def dispatch(self, request, *args, **kwargs):
        return super(MessageSend, self).dispatch(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        p = {'access_token': request.GET.get('access_token')}
        if request.GET.get('threadId'):
            p['threadId'] = request.GET.get('threadId')
        if request.GET.get('key'):
            p['key'] = request.GET.get('key')
        new_key = cache.get('max_value', 1)
        cache.set('max_value', new_key+1)

        msg = base64.urlsafe_b64decode(simplejson.loads(request.body)['raw'])
        msg = email.message_from_string(msg)
        if msg.get_default_type() == 'text/plain':
            n_msg = MIMEMultipart('alternative')
            n_msg['To'] = msg['To']
            n_msg['From'] = msg['From']
            n_msg['Subject'] = msg['Subject']
            n = msg.get_payload()
            n = quopri.decodestring(n).decode('utf-8')
            logging.debug(u'MSG: {}'.format(n))
            n = n + u'<img width="1" height="1" src="http://lab.mailburn.com/track.gif?m=mail-{}" />'.format(new_key)
            n = n.encode('utf-8')
            n_text = MIMEText(n, 'html')
            n_text.set_charset('utf-8')
            n_msg.attach(n_text)
        else:
            n_msg = msg
#        raise KeyError(n_msg.as_string())
#        msg.set_default_type('text/html')

        d = {'raw': base64.urlsafe_b64encode(n_msg.as_string())}
        d['raw'] = d['raw'].replace('+', '-')
        d = simplejson.dumps(d)
        r = requests.post('https://www.googleapis.com/gmail/v1/users/me/messages/send',
                        params=p, data=d, headers={
                'Authorization': request.META['HTTP_AUTHORIZATION'],
                'Content-Type': 'application/json'
            })
        if r.status_code == 200:
            cache.set('mail-{}'.format(new_key), r.json()['id'])
            return HttpResponse(json.dumps(r.json()), content_type='application/json')
        else:
            return HttpResponse(r.text, content_type='application/json', status=r.status_code)

class TrackView(View):
    def get(self, request, *args, **kwargs):
        if request.GET.get('m'):
            if cache.get(request.GET.get('m')):
                cache.set(cache.get(request.GET.get('m')), 'Opened')
        r = HttpResponse('')
        r['Cache-Control'] = 'no-cache'
        r['Content-Length'] = 0
        return r


import sys
import os
import zipfile
import numpy as np
import pandas as pd
from geopandas.geodataframe import GeoDataFrame
from shapely import wkt

# Assuming this is either running in the root directory or opd-dev inside the root directory
if os.path.basename(os.getcwd()) == "openpolicedata":
    sys.path.append(os.path.join("..","openpolicedata"))
    output_dir = os.path.join(".","data","backup")
else:
    sys.path.append(os.path.join("..","..","openpolicedata"))
    output_dir = os.path.join("..","data","backup")
import openpolicedata as opd
from datetime import datetime

istart = 510
update = None #"changes"
include_stanford = False
src_file = "..\opd-data\opd_source_table.csv"

if src_file != None:
    opd.datasets.datasets = opd.datasets._build(src_file)

datasets = opd.datasets.query()

print(f"Output directory: {output_dir}")
log_filename = f"DataBackup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
log_folder = os.path.join(output_dir,"backup_logs")
if not os.path.exists(output_dir):
    os.mkdir(output_dir)
if not os.path.exists(log_folder):
    os.mkdir(log_folder)

def log_to_file(*args):
    filename = os.path.join(log_folder, log_filename)
    with open(filename, "a") as f:
        for x in args:
            if hasattr(x, '__iter__') and type(x) != str:
                new_line = ', '.join([str(y) for y in x])
            else:
                new_line = str(x)

            now = datetime.now().strftime("%d.%b %Y %H:%M")
            f.write(f"{now}: {new_line}\n")

log_to_file("Beginning data backup")

this_year = datetime.now().year

def check_equality(df, df_new, year, is_sorted):
    # is_sorted is helpful for debug breakpoints
    is_equal = True
    for i in range(len(df_new.columns)):
        if "location" in df_new.columns[i].lower() or \
            "latitude" in df_new.columns[i].lower() or \
            "longitude" in df_new.columns[i].lower() or \
            df_new.columns[i] in ['data_loaded_at']:  # date_loaded_at can get updated if they reload
            # Locations can be dicts that can get mangled in CSV conversion
            continue
        a = df.iloc[:,i]
        b = df_new.iloc[:,i]
        if not a.eq(b).all():
            if a.dtype == float:
                if (a-b).abs().max() >= 1e-10:
                    # For debugging
                    diffs = [(a.iloc[k],b.iloc[k]) for k in range(len(df)) if a.iloc[k] != b.iloc[k]]
                    diffs = [x for x in diffs if ((pd.notnull(x[0]) and x[0]!="") or (pd.notnull(x[1]) and x[1]!=""))]
                    if len(diffs)>0:
                        is_equal = False
                        break
            else:
                diffs = [(k,a.iloc[k],b.iloc[k]) for k in range(len(df)) if a.iloc[k] != b.iloc[k]]
                diffs = [x for x in diffs if ((pd.notnull(x[1]) and x[1]!="") or (pd.notnull(x[2]) and x[2]!=""))]
                if (year in [opd.defs.NA, opd.defs.MULTI] or year >= this_year-1) and len(diffs)>0:
                    # This could be something that has been completed like a use of force investigation
                    diffs = [x for x in diffs if (x[1] not in ["NO REPORT"] or x[2] not in ["REPORT"])]
                if len(diffs)>0:
                    # Sometimes numbers are loaded as strings from CSV or vise versa. Try comparing strings
                    diffs = [(k,a.iloc[k],b.iloc[k]) for k in range(len(df)) if str(a.iloc[k]).lower() != str(b.iloc[k]).lower()]
                    diffs = [x for x in diffs if ((pd.notnull(x[1]) and x[1]!="") or (pd.notnull(x[2]) and x[2]!=""))]
                if len(diffs)>0:
                    is_equal = False
                    break

    return is_equal

def compare_dfs(zipname, table, dataset, year):
    df = pd.read_csv(zipname).replace("#N/A",np.nan).replace("NA",np.nan).replace("N/A",np.nan).replace("NULL",np.nan).replace("n/a",np.nan)
    if pd.notnull(dataset["date_field"]):
        df = df.astype({dataset["date_field"]: 'datetime64[ns]'})
    df_new = table.table.replace("#N/A",np.nan).replace("NA",np.nan).replace("N/A",np.nan).replace("NULL",np.nan).replace("n/a",np.nan)
    if type(df_new) == GeoDataFrame:
        df['geometry'] = df['geometry'].apply(wkt.loads)
        df = GeoDataFrame(df, crs=df_new.crs)
        df.pop("geometry")
        df_new.pop("geometry")

    for col in df_new.columns:
        if df_new[col].dtype == "object" and df[col].dtype != "object":
            if df[col].dtype == bool:
                df_new[col] = df_new[col].map({"False" : False, "True" : True, "false" : False, "true" : True, "FALSE" : False, "TRUE" : True}, na_action="ignore")
            else:
                if df[col].dtype==float:
                    df_new[col] = df_new[col].replace("",np.nan)
                elif df[col].dtype == np.int64:
                    df[col] = df[col].astype(pd.Int64Dtype())
                df_new[col] = df_new[col].astype(df[col].dtype)
        elif df_new[col].dtype != "object" and df[col].dtype == "object":
            if df_new[col].dtype==float:
                df[col] = df[col].replace("",np.nan)
            df[col] = df[col].astype(df_new[col].dtype)

    df = df[df_new.columns]

    is_equal = df.eq(df_new).all().all()
    if not is_equal and len(df_new) == len(df):
        is_equal = check_equality(df, df_new, year, False)

        if not is_equal:
            # sortby = []
            # if pd.notnull(dataset["date_field"]) and dataset["date_field"] not in sortby:
            #     sortby = [dataset["date_field"]]
            # sortby.extend(df.columns[0:3])
            df.sort_values(by=list(df.columns), inplace=True, ignore_index=True, key=lambda col: col.apply(lambda x: str(x).lower()))
            df_new.sort_values(by=list(df.columns), inplace=True, ignore_index=True, key=lambda col: col.apply(lambda x: str(x).lower()))
            is_equal = check_equality(df, df_new, year, True)

    if is_equal:
        pass
    elif len(df_new) == len(df):
        pass
    elif (year==opd.defs.NA or year==opd.defs.MULTI or year >= this_year-1) and len(df_new) <= len(df):
        raise ValueError("Data has gotten shorter for recent data")
    elif year!=opd.defs.NA and year!=opd.defs.MULTI and year < this_year-1:
        raise("Older data has changed")

    return is_equal

for i in range(istart, len(datasets)):
    if not include_stanford and "stanford" in datasets.iloc[i]["URL"]:
        continue
    srcName = datasets.iloc[i]["SourceName"]
    state = datasets.iloc[i]["State"]

    table_print = datasets.iloc[i]["TableType"]
    now = datetime.now().strftime("%d.%b %Y %H:%M:%S")
    print(f"{now} Saving CSV for dataset {i} of {len(datasets)}: {srcName} {table_print} table")

    src = opd.Source(srcName, state=state)

    if datasets.iloc[i]["DataType"] in [opd.defs.DataType.CSV.value, opd.defs.DataType.EXCEL.value] or \
        datasets.iloc[i]["Year"] != opd.defs.MULTI:
        csv_filename = src.get_csv_filename(datasets.iloc[i]["Year"], output_dir, datasets.iloc[i]["TableType"])
        zipname = csv_filename.replace(".csv",".zip")

        update_data = False
        if os.path.exists(zipname):
            if update == "changes":
                update_data = True
            else:
                continue

        try:
            table = src.load_from_url(datasets.iloc[i]["Year"], datasets.iloc[i]["TableType"])
        except Exception as e:
            log_to_file(srcName, datasets.iloc[i]["TableType"], datasets.iloc[i]["Year"], e.args)
            continue

        if update_data and compare_dfs(zipname, table, datasets.iloc[i], datasets.iloc[i]["Year"]):
            continue

        csv_filename = table.to_csv(output_dir)
        zipname = csv_filename.replace(".csv",".zip")
        with zipfile.ZipFile(zipname, mode="w", compression=zipfile.ZIP_LZMA) as archive:
            archive.write(csv_filename)
        if not os.path.exists(zipname):
            raise FileExistsError(zipname)
        os.remove(csv_filename)
    else:
        try:
            years = src.get_years(datasets.iloc[i]["TableType"])
        except Exception as e:
            log_to_file(srcName, datasets.iloc[i]["TableType"], datasets.iloc[i]["Year"], e.args)
            continue

        now = datetime.now().strftime("%d.%b %Y %H:%M:%S")
        print(f"\t{now} Years: {years}")

        for year in years:
            csv_filename = src.get_csv_filename(year, output_dir, datasets.iloc[i]["TableType"])

            zipname = csv_filename.replace(".csv",".zip")
            update_data = False
            if os.path.exists(zipname):
                if update == "changes":
                    update_data = True
                else:
                    continue

            print(f"\t{now} Year: {year}")

            try:
                table = src.load_from_url(year, datasets.iloc[i]["TableType"])
            except Exception as e:
                log_to_file(srcName, datasets.iloc[i]["TableType"], datasets.iloc[i]["Year"], year, e.args)
                continue

            if update_data and compare_dfs(zipname, table, datasets.iloc[i], year):
                continue

            csv_filename = table.to_csv(output_dir)
            zipname = csv_filename.replace(".csv",".zip")
            with zipfile.ZipFile(zipname, mode="w", compression=zipfile.ZIP_LZMA) as archive:
                archive.write(csv_filename)
            if not os.path.exists(zipname):
                raise FileExistsError(zipname)
            os.remove(csv_filename)

log_to_file("Completed data backup")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import numpy as np
import pandas as pd

###load numpy datas
data_dict = {}

path = '../processedData'
dirs = os.listdir(path)
for x in dirs:
    if os.path.splitext(x)[1] == ".npy":
        filePath = path + '/' + x
        data_key = os.path.splitext(x)[0]
        data_value = np.load(filePath)
        data_dict[data_key] = data_value

sensor_onset_dict = {'S01_20210419_103410_110410_C1': 1618794899,
                     'S01_20210423_101906_104906_C1': 1619140688,
                     'S01_20210428_091333_093006_C2': 1619572354}

def datetime_to_timestamp(datetime):
    datetime = pd.to_datetime(datetime)
    timestamp = (datetime - pd.Timestamp("1970-01-01")) // pd.Timedelta('1s')
    return timestamp-28800

def read_segment(filename):
    time_parts = filename.split('_')
    date_str = time_parts[1]
    onset_str = time_parts[2]
    offset_str = time_parts[3]
    date = date_str[0:4] + '-' + date_str[4:6] + '-' + date_str[6:8]
    onset = onset_str[0:2] + ':' + onset_str[2:4] + ':' + onset_str[4:6]
    offset = offset_str[0:2] + ':' + offset_str[2:4] + ':' + offset_str[4:6]
    onset_datetime_str = date + ' ' + onset
    offset_datetime_str = date + ' ' + offset
    return onset_datetime_str, offset_datetime_str

def deal_20210420_1106(filename):
    onset_offset = [('2021-04-20 11:06:47', '2021-04-20 11:16:47'),
                    ('2021-04-20 11:26:48', '2021-04-20 11:36:48')]
    sensor_onset = sensor_onset_dict[filename]
    old_data = data_dict[filename]
    new_data = []
    for onset, offset in onset_offset:
        start_idx = datetime_to_timestamp(onset) - sensor_onset
        end_idx = datetime_to_timestamp(offset) - sensor_onset

        if start_idx < 0:
            start_idx = 0
        if end_idx >= len(old_data):
            end_idx = len(old_data) - 1

        if len(new_data) == 0:
            new_data = old_data[start_idx:end_idx]
        else:
            new_data = np.vstack((new_data, old_data[start_idx:end_idx]))
    np.save('../codedData/'+filename+'.npy', new_data)

def deal_20210426_0859(filename):
    onset_offset = [('2021-04-26 10:10:47', '2021-04-26 10:25:47'),
                    ('2021-04-26 10:58:18', '2021-04-26 11:13:18')]
    sensor_onset = sensor_onset_dict[filename]
    old_data = data_dict[filename]
    new_data = []
    for onset, offset in onset_offset:
        start_idx = datetime_to_timestamp(onset) - sensor_onset
        end_idx = datetime_to_timestamp(offset) - sensor_onset

        if start_idx < 0:
            start_idx = 0
        if end_idx >= len(old_data):
            end_idx = len(old_data) - 1

        if len(new_data) == 0:
            new_data = old_data[start_idx:end_idx]
        else:
            new_data = np.vstack((new_data, old_data[start_idx:end_idx]))
    np.save('../codedData/'+filename+'.npy', new_data)

for datafile in data_dict.keys():
    if datafile == '20210420_1106':
        deal_20210420_1106(datafile)
        continue
    if datafile == '20210426_0859':
        deal_20210426_0859(datafile)
        continue

    onset, offset = read_segment(datafile)
    onset_timestamp = datetime_to_timestamp(onset)
    offset_timestamp = datetime_to_timestamp(offset)
    sensor_onset = sensor_onset_dict[datafile]
    start_idx = onset_timestamp - sensor_onset
    end_idx = offset_timestamp - sensor_onset
    old_data = data_dict[datafile]

    if start_idx < 0:
        start_idx = 0
    if end_idx >= len(old_data):
        end_idx = len(old_data) - 1

    new_data = old_data[start_idx:end_idx+1]
    np.save('../codedData/'+datafile+'.npy', new_data)