"""
PatchCore logic based on https://github.com/rvorias/ind_knn_ad
"""

from sklearn import random_projection
from utils.utils import KNNGaussianBlur
from utils.utils import set_seeds
import numpy as np
from sklearn.metrics import roc_auc_score
import timm
import torch
from tqdm import tqdm
from utils.au_pro_util import calculate_au_pro

class Features(torch.nn.Module):

    def __init__(self, image_size=224, f_coreset=0.1, coreset_eps=0.9):
        super().__init__()
        self.device = "cuda" if torch.cuda.is_available() else "cpu"
        self.deep_feature_extractor = Model(device=self.device)
        self.deep_feature_extractor.to(self.device)
        self.deep_feature_extractor.freeze_parameters(layers=[], freeze_bn=True)

        self.image_size = image_size
        self.f_coreset = f_coreset
        self.coreset_eps = coreset_eps
        self.average = torch.nn.AvgPool2d(3, stride=1)
        self.blur = KNNGaussianBlur(4)
        self.n_reweight = 3
        set_seeds(0)
        self.patch_lib = []
        self.resize = torch.nn.AdaptiveAvgPool2d((28, 28))

        self.image_preds = list()
        self.image_labels = list()
        self.pixel_preds = list()
        self.pixel_labels = list()
        self.gts = []
        self.predictions = []
        self.image_rocauc = 0
        self.pixel_rocauc = 0
        self.au_pro = 0

    def __call__(self, x):
        # Extract the desired feature maps using the backbone model.
        with torch.no_grad():
            feature_maps = self.deep_feature_extractor(x)

        feature_maps = [fmap.to("cpu") for fmap in feature_maps]
        return feature_maps

    def add_sample_to_mem_bank(self, sample):
        raise NotImplementedError

    def predict(self, sample, mask, label):
        raise NotImplementedError

    def compute_s_s_map(self, patch, feature_map_dims, mask, label):
        dist = torch.cdist(patch, self.patch_lib)
        min_val, min_idx = torch.min(dist, dim=1)
        s_idx = torch.argmax(min_val)
        s_star = torch.max(min_val)

        # reweighting
        m_test = patch[s_idx].unsqueeze(0)  # anomalous patch
        m_star = self.patch_lib[min_idx[s_idx]].unsqueeze(0)  # closest neighbour
        w_dist = torch.cdist(m_star, self.patch_lib)  # find knn to m_star pt.1
        _, nn_idx = torch.topk(w_dist, k=self.n_reweight, largest=False)  # pt.2
        # equation 7 from the paper
        m_star_knn = torch.linalg.norm(m_test - self.patch_lib[nn_idx[0, 1:]], dim=1)
        # Softmax normalization trick as in transformers.
        # As the patch vectors grow larger, their norm might differ a lot.
        # exp(norm) can give infinities.
        D = torch.sqrt(torch.tensor(patch.shape[1]))
        w = 1 - (torch.exp(s_star / D) / (torch.sum(torch.exp(m_star_knn / D))))
        s = w * s_star

        # segmentation map
        s_map = min_val.view(1, 1, *feature_map_dims)
        s_map = torch.nn.functional.interpolate(s_map, size=(self.image_size, self.image_size), mode='bilinear')
        s_map = self.blur(s_map)

        self.image_preds.append(s.numpy())
        self.image_labels.append(label)
        self.pixel_preds.extend(s_map.flatten().numpy())
        self.pixel_labels.extend(mask.flatten().numpy())
        self.predictions.append(s_map.detach().cpu().squeeze().numpy())
        self.gts.append(mask.detach().cpu().squeeze().numpy())

    def calculate_metrics(self):
        self.image_preds = np.stack(self.image_preds)
        self.image_labels = np.stack(self.image_labels)
        self.pixel_preds = np.array(self.pixel_preds)

        self.image_rocauc = roc_auc_score(self.image_labels, self.image_preds)
        self.pixel_rocauc = roc_auc_score(self.pixel_labels, self.pixel_preds)
        self.au_pro, _ = calculate_au_pro(self.gts, self.predictions)

    def run_coreset(self):
        self.patch_lib = torch.cat(self.patch_lib, 0)
        if self.f_coreset < 1:
            self.coreset_idx = self.get_coreset_idx_randomp(self.patch_lib,
                                                            n=int(self.f_coreset * self.patch_lib.shape[0]),
                                                            eps=self.coreset_eps, )
            self.patch_lib = self.patch_lib[self.coreset_idx]

    def get_coreset_idx_randomp(self, z_lib, n=1000, eps=0.90, float16=True, force_cpu=False):
        """Returns n coreset idx for given z_lib.
        Performance on AMD3700, 32GB RAM, RTX3080 (10GB):
        CPU: 40-60 it/s, GPU: 500+ it/s (float32), 1500+ it/s (float16)
        Args:
            z_lib:      (n, d) tensor of patches.
            n:          Number of patches to select.
            eps:        Agression of the sparse random projection.
            float16:    Cast all to float16, saves memory and is a bit faster (on GPU).
            force_cpu:  Force cpu, useful in case of GPU OOM.
        Returns:
            coreset indices
        """

        print(f"   Fitting random projections. Start dim = {z_lib.shape}.")
        try:
            transformer = random_projection.SparseRandomProjection(eps=eps)
            z_lib = torch.tensor(transformer.fit_transform(z_lib))
            print(f"   DONE.                 Transformed dim = {z_lib.shape}.")
        except ValueError:
            print("   Error: could not project vectors. Please increase `eps`.")

        select_idx = 0
        last_item = z_lib[select_idx:select_idx + 1]
        coreset_idx = [torch.tensor(select_idx)]
        min_distances = torch.linalg.norm(z_lib - last_item, dim=1, keepdims=True)
        # The line below is not faster than linalg.norm, although i'm keeping it in for
        # future reference.
        # min_distances = torch.sum(torch.pow(z_lib-last_item, 2), dim=1, keepdims=True)

        if float16:
            last_item = last_item.half()
            z_lib = z_lib.half()
            min_distances = min_distances.half()
        if torch.cuda.is_available() and not force_cpu:
            last_item = last_item.to("cuda")
            z_lib = z_lib.to("cuda")
            min_distances = min_distances.to("cuda")

        for _ in tqdm(range(n - 1)):
            distances = torch.linalg.norm(z_lib - last_item, dim=1, keepdims=True)  # broadcasting step
            min_distances = torch.minimum(distances, min_distances)  # iterative step
            select_idx = torch.argmax(min_distances)  # selection step

            # bookkeeping
            last_item = z_lib[select_idx:select_idx + 1]
            min_distances[select_idx] = 0
            coreset_idx.append(select_idx.to("cpu"))
        return torch.stack(coreset_idx)

class Model(torch.nn.Module):

    def __init__(self, device, backbone_name='wide_resnet50_2', out_indices=(2, 3), checkpoint_path='',
                 pool_last=False):
        super().__init__()
        # Determine if to output features.
        kwargs = {'features_only': True if out_indices else False}
        if out_indices:
            kwargs.update({'out_indices': out_indices})

        self.backbone = timm.create_model(model_name=backbone_name, pretrained=True, checkpoint_path=checkpoint_path,
                                          **kwargs)
        self.device = device
        self.avg_pool = torch.nn.AdaptiveAvgPool2d((1, 1)) if pool_last else None

    def forward(self, x):
        x = x.to(self.device)

        # Backbone forward pass.
        features = self.backbone(x)

        # Adaptive average pool over the last layer.
        if self.avg_pool:
            fmap = features[-1]
            fmap = self.avg_pool(fmap)
            fmap = torch.flatten(fmap, 1)
            features.append(fmap)

        return features

    def freeze_parameters(self, layers, freeze_bn=False):
        """ Freeze resent parameters. The layers which are not indicated in the layers list are freeze. """

        layers = [str(layer) for layer in layers]
        # Freeze first block.
        if '1' not in layers:
            if hasattr(self.backbone, 'conv1'):
                for p in self.backbone.conv1.parameters():
                    p.requires_grad = False
            if hasattr(self.backbone, 'bn1'):
                for p in self.backbone.bn1.parameters():
                    p.requires_grad = False
            if hasattr(self.backbone, 'layer1'):
                for p in self.backbone.layer1.parameters():
                    p.requires_grad = False

        # Freeze second block.
        if '2' not in layers:
            if hasattr(self.backbone, 'layer2'):
                for p in self.backbone.layer2.parameters():
                    p.requires_grad = False

        # Freeze third block.
        if '3' not in layers:
            if hasattr(self.backbone, 'layer3'):
                for p in self.backbone.layer3.parameters():
                    p.requires_grad = False

        # Freeze fourth block.
        if '4' not in layers:
            if hasattr(self.backbone, 'layer4'):
                for p in self.backbone.layer4.parameters():
                    p.requires_grad = False

        # Freeze last FC layer.
        if '-1' not in layers:
            if hasattr(self.backbone, 'fc'):
                for p in self.backbone.fc.parameters():
                    p.requires_grad = False

        if freeze_bn:
            for module in self.backbone.modules():
                if isinstance(module, torch.nn.modules.BatchNorm1d):
                    module.eval()
                if isinstance(module, torch.nn.modules.BatchNorm2d):
                    module.eval()
                if isinstance(module, torch.nn.modules.BatchNorm3d):
                    module.eval()


# coding=utf-8

from flask import jsonify, request, g, url_for
from .. import db
from ..models.models import Post, Permission
from ..models.manager import PostManager, UserManager
from . import api
from .decorators import permission_required
from .errors import forbidden
from .authentication import auth

@api.route('/posts/')
@auth.login_required
def get_posts():
    page = request.args.get('page', 1, type=int)
    pagination = Post.query.paginate(page, per_page=10, error_out=False)
    posts = pagination.items
    prev = None
    if pagination.has_prev:
        prev = url_for('api.get_posts', page=page-1, _external=True)
    next = None
    if pagination.has_next:
        next = url_for('api.get_posts', page=page+1, _external=True)
    return jsonify({
        'posts': [PostManager.to_json(post) for post in posts],
        'prev': prev,
        'next': next,
        'count': pagination.total
    })

@api.route('/posts/<int:id>')
@auth.login_required
def get_post(id):
    post = Post.query.get_or_404(id)
    return jsonify(PostManager.to_json(post))

@api.route('/posts/', methods=['POST'])
@permission_required(Permission.WRITE_ARTICLES)
def new_post():
    PostManager.add_post(head=request.form.get('head'), body=request.form.get('body'),
                         author=g.current_user)
    return jsonify({'success': True})

@api.route('/posts/<int:id>', methods=['POST'])
@permission_required(Permission.WRITE_ARTICLES)
def edit_post(id):
    post = Post.query.get_or_404(id)
    if g.current_user != post.author and \
            not UserManager.can(g.current_user, Permission.ADMINISTER):
        return forbidden('Insufficient permissions')
    post.title = request.form.get('head', post.head)
    post.body = request.form.get('body', post.body)
    db.session.add(post)
    return jsonify(PostManager.to_json(post))


# Create your views here.
from django.template import Context, loader
from django.http import HttpResponse

def hello_world(request):
    todos = [
        {
            'title': 'Mow the lawn',
            'importance': 'Minor'
        },
        {
            'title': 'Backup your PC',
            'importance': 'High'
        },
        {
            'title': 'Buy some Milk',
            'importance': 'Medium'
        },
        {
            'title': 'See the film',
            'importance': 'High'
        }
    ]
    t = loader.get_template('index.tmpl')
    c = Context({
        'todos': todos,
    })

    return HttpResponse(t.render(c))


#! /usr/bin/env python3

from Bio import SeqIO
from argparse import ArgumentParser

parser = ArgumentParser(description='Filter FASTA file by the percentage of X and lowercase letters')
parser.add_argument('-f', type=str, help='FASTA file')
parser.add_argument('-n', type=float, help='Max acceptable proportion of X and lowercase')
args = parser.parse_args()

def lowercase_count(line):
    """
    A count of lowercase letters and/or X'es in a string
    :param line:
    :return:
    """
    count = 0
    for letter in line:
        if letter.islower() or letter == 'X':
            count += 1
    return count

with open('{}.filtered'.format(args.f), mode='w') as outfasta:
    for record in SeqIO.parse(args.f, 'fasta'):
        if lowercase_count(record.seq)/len(record) <= args.n:
            SeqIO.write(record, outfasta, 'fasta')


import matplotlib.pyplot as plt
import numpy as np
np.set_printoptions(precision=2)
import sklearn.metrics as skm
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
from summ_evaluator import SummaryEvaluator

class ClassificationEvaluator(SummaryEvaluator):
    def __init__(self, ds_name, model_name, num_classes):
        super().__init__(ds_name, model_name, num_classes)

        #metrics
        self.report_dict = None

        self.precision = None
        self.recall = None
        self.f1_score = None
        self.support = None

        self.true_negatives = None
        self.false_positives = None
        self.false_negatives = None
        self.true_positives = None

    def get_metrics(self, random=False, threshold=None, print_output=False, colorbar=False):
        y = self.get_y(random, threshold)

        self.report_dict = skm.classification_report(self.y_true, y, output_dict=True)
        self.precision = self.report_dict['1']['precision']
        self.recall = self.report_dict['1']['recall']
        self.f1_score = self.report_dict['1']['f1-score']
        self.support = self.report_dict['1']['support']
        if print_output:
            print(self.report_dict)

        tn, fp, fn, tp = skm.confusion_matrix(self.y_true, y).ravel()
        self.true_negatives = tn
        self.false_positives = fp
        self.false_negatives = fn
        self.true_positives = tp
        if print_output:
            print('tn: {}, fp: {}, fn: {}, tp: {}'.format(tn, fp, fn, tp))

        return skm.f1_score(self.y_true, y)

    def plot(self, normalize=False, title=None, cmap=plt.cm.Blues, random=False, threshold=None, colorbar=False):
        y = self.get_y(random, threshold)
        """
        This function prints and plots the confusion matrix.
        Normalization can be applied by setting `normalize=True`.
        """

        if not title:
            if normalize:
                title = 'Normalized confusion matrix'
            else:
                title = 'Confusion matrix, {}'.format(self.ds_name)

        # Compute confusion matrix
        cm = confusion_matrix(self.y_true, y)
        # Only use the labels that appear in the data
        self.class_names = ['0', '1']

        if normalize:
            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            print("Normalized confusion matrix")
        else:
            print('Confusion matrix, {}'.format(self.ds_name))

        fig, ax = plt.subplots()
        im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
        if colorbar:
            ax.figure.colorbar(im, ax=ax)
        # We want to show all ticks...
        ax.set(xticks=np.arange(cm.shape[1]),
               yticks=np.arange(cm.shape[0]),
               # ... and label them with the respective list entries
               xticklabels=self.class_names, yticklabels=self.class_names,
               ylabel='True label',
               xlabel='Predicted label')

        ax.yaxis.label.set_size('xx-large')
        ax.xaxis.label.set_size('xx-large')
        # Rotate the tick labels and set their alignment.
        plt.setp(ax.get_xticklabels(), ha="center")  # , rotation=45, rotation_mode="anchor")
        for i, t in enumerate(ax.get_xticklabels()):
            t.set_fontsize('xx-large')
            #plt.setp(ax.get_yticklabels(), ha="center", rotation=90, rotation_mode="anchor")
        for i, t in enumerate(ax.get_yticklabels()):
            t.set_fontsize('xx-large')
            t.set_rotation('vertical')
            if i == 0:
                ha = 'right'
                t.set_ha(ha)
            else:
                ha = 'right'
                t.set_ha(ha)

        # Loop over data dimensions and create text annotations.
        fmt = '.2f' if normalize else 'd'
        thresh = cm.max() / 2.
        for i in range(cm.shape[0]):
            if i == 0:
                va = 'top'
                y = i + 0.2
            else:
                va = 'bottom'
                y = i - 0.2
            for j in range(cm.shape[1]):
                ax.text(j, y, format(cm[i, j], fmt),
                        ha="center", va=va, size='xx-large',
                        color="white" if cm[i, j] > thresh else "black")
        fig.tight_layout()

        plt.savefig('/home/emma/summary_evaluation/results/confusion_{}.pdf'.format(self.ds_name))

if __name__ == '__main__':

    class_eval = ClassificationEvaluator('run_1_two', num_classes=2)

    class_eval.get_metrics()

    # Plot non-normalized confusion matrix
    class_eval.plot(title='Confusion matrix, without normalization')

    # Plot normalized confusion matrix
    class_eval.plot(normalize=True, title='Normalized confusion matrix')

    plt.show()


from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import MyHelper
import FileOperator

f = open(".\FeatureVectors.txt",'a')
f.seek(0)
f.truncate()
f.close()

print("-----trees-----")
tree_array=np.zeros((10,2))#创建
for i in range(1,11):

    p= np.loadtxt("./points/new_trees/tree"+str(i)+".txt")
    print('No.',str(i),MyHelper.GetFeatureVector(p))
    v = MyHelper.GetFeatureVector(p)
    tree_array[i-1,:]=v[0,:]
    tree=plt.scatter(v[0,0],v[0,1],c = 'g')

    FileOperator.WriteData(".\FeatureVectors.txt",v,1)

print('tree_array.shape: ',tree_array.shape)

print("-----buildings-----")
building_array=np.zeros((10,2))
for i in range(1,11):
    p= np.loadtxt("./points/new_building/building"+str(i)+".txt")
    print('No.',str(i),MyHelper.GetFeatureVector(p))
    v = MyHelper.GetFeatureVector(p)
    tree_array[i-1, :] = v[0,:]
    # print(tree_array[i-1,:])
    building=plt.scatter(v[0,0],v[0,1],c = 'r')
    FileOperator.WriteData(".\FeatureVectors.txt",v,-1)

print('building_array.shape: ',building_array.shape)
plt.legend([tree,building],['tree','building'],)

#绘制
# plt.figure()
# tree=plt.scatter(tree_array[:,0],tree_array[:,1],c='g')
# building=plt.scatter(building_array[:,0],building_array[:,1],c='r')
# plt.legend([tree,building],['tree','building'])

plt.show()


import socket
import pickle
import threading
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = ''
port = 4579
s.bind((host,port))
s.listen(2)
client = []
chunk_size = 4096
def recv_data(conn):
    while True:
        data = b""
        data_size = 0
        size = conn.recv(10).decode('UTF-8')
        print("Size recieved")
        while data_size < int(size):
            if(data_size+chunk_size) > int(size):
                re = conn.recv(int(size)-data_size)
            else:
                re = conn.recv(chunk_size)
            data+=re
            data_size+=len(re)
        print("All Data recieved")
        for c in client:
            if c!= conn:
                c.send(size.encode('UTF-8'))
                c.send(data)
                print("Data sent to client")
        print("Done")
    conn.close()

while True:
    print("Waiting for Connections.....")
    conn, adr = s.accept()
    client.append(conn)
    print("Connection from "+str(adr))
    threading.Thread(target=recv_data,args=(conn,)).start()


import socket
import asyncio
from aiohttp import ClientSession
import ipaddress as ip_addr
import netifaces
import requests
from aiohttp.client_exceptions import ClientConnectorError
from contextlib import suppress

class Scanner:

    def get_net_addr(self, ip, mask):
        return ip_addr.ip_address(int(ip) & int(mask))

    def get_ip(self):
        interfaces = netifaces.interfaces()
        for i in interfaces:
            if i == 'lo':
                continue
            iface = netifaces.ifaddresses(i).get(netifaces.AF_INET)
            if iface != None:
                for j in iface:
                    myIP = (ip_addr.ip_address(j['addr']),ip_addr.ip_address( j['netmask']))
        return myIP

    def addresses_in_subnet(self, host_ip, mask):
        return [i for i in ip_addr.ip_network(str(host_ip) + '/' + str(mask)).hosts()]

    def scan_all(self):
        loop = asyncio.get_event_loop()
        (ip, mask) = self.get_ip()
        addresses = self.addresses_in_subnet(self.get_net_addr(ip, mask), mask)
        addresses.remove(ip)

        app_instances = []

        future = asyncio.ensure_future(self.check_all(addresses))
        responses =loop.run_until_complete(future)
        result = []
        for response in responses:
            if not response is None and response[1] == 'hey':
                result.append(response)

        return result

    async def fetch(self, ip_address, session):
        with suppress(ClientConnectorError):
            async with session.get('http://' +str(ip_address) + ':5000/greet') as response:
                return (ip_address,await  response.text())

    async def check_all(self, ip_addresses):
        tasks= []

        async with ClientSession() as session:
            for address in ip_addresses:
                task = asyncio.ensure_future(self.fetch(address, session))
                tasks.append(task)

            responses = await asyncio.gather(*tasks)
        return responses

    def check(self, ip_address, session):
        try:
            r = session.get('http://' +str(ip_address) + ':5000/greet', timeout=1 )
            if r.text == 'hey':
                return (ip_address, True)
            else:
                return (ip_address, False)
        except:
            return (ip_address, False)


from datetime import datetime, date
from typing import FrozenSet, List, Dict

INT_LENGTH = 9
STR_LENGTH = 9

INT_LIST_LENGTH = 5

MIN_YEAR = 0
MAX_YEAR = 2200
MIN_MONTH = 1
MAX_MONTH = 12
MIN_DAY = 1
MAX_DAY = 31
MIN_HOUR = 0
MAX_HOUR = 23
MIN_MINUTE = 0
MAX_MINUTE = 59
MIN_SECOND = 1
MAX_SECOND = 59

def supported_types() -> FrozenSet:
    return frozenset([int, bool, str, datetime, date, List[int], Dict[str, int], dict])


import sys

lookdir = {0: [-1, 0], 1: [0, 1], 2: [1, 0], 3: [0, -1]}
n, m = map(int, sys.stdin.readline().split())
x_, y_, lookat = map(int, sys.stdin.readline(). split())
map_ = [[] for _ in range(n)]
for idx in range(n):
    map_[idx] = list(map(int, sys.stdin.readline().split()))
map_[x_][y_] = 3

rot_idx = 0
ans = 1
while True:
    lookat = (lookat + 1) if lookat != 3 else 0
    x_new, y_new = x_ + lookdir[lookat][0], y_ + lookdir[lookat][1]
    if map_[x_new][y_new] == 0:
        x_, y_ = x_new, y_new
        map_[x_][y_] = 3
        rot_idx = 0
        ans += 1
    elif rot_idx == 4:
        x_new, y_new = x_ - lookdir[lookat][0], y_ - lookdir[lookat][1]
        if map_[x_new][y_new] == 1:
            break
        else:
            x_, y_ = x_new, y_new
            rot_idx = 0
    else:
        rot_idx += 1
print(ans)


n = input()
l = [int(i) for i in input().strip().split(' ')]
l.append(-1)

def seq(l):
    sum = 0
    max = 0
    for i in range(0, len(l), 1):
        if l[i] == -1:
            break
        if l[i] < l[i + 1]:
            sum = sum + l[i]
        else:
            sum = sum + l[i]
            if max < sum:
                max = sum
            sum = 0
    print(max)

seq(l)


"""Early-pipeline operations that error-check and lower grammars."""

from __future__ import annotations
# mypy: disallow-untyped-defs, disallow-incomplete-defs, disallow-untyped-calls

import collections
import dataclasses
from dataclasses import dataclass
import typing

from .grammar import (CallMethod, Element, End, ErrorSymbol, Exclude, Grammar,
                      LenientNt, Literal, LookaheadRule, Optional,
                      NoLineTerminatorHere, Nt, NtDef, NtParameter, Production,
                      ReduceExpr, ReduceExprOrAccept, Some, UnicodeCategory,
                      Var, is_concrete_element)
from .ordered import OrderedFrozenSet, OrderedSet
from .runtime import ErrorToken, ErrorTokenClass

# *** Checking for errors *****************************************************

T = typing.TypeVar("T")

def fix(f: typing.Callable[[T], T], start: T) -> T:
    """Compute a fixed point of `f`, the hard way, starting from `start`."""
    prev, current = start, f(start)
    while current != prev:
        prev, current = current, f(current)
    return current

def empty_nt_set(grammar: Grammar) -> typing.Dict[LenientNt, ReduceExprOrAccept]:
    """Determine which nonterminals in `grammar` can produce the empty string.

    Return a dict {nt: expr} that maps each such nonterminal to the expr
    that should be evaluated when reducing the empty string to nt.
    So, for example, if we have a production

        a ::= b? c?  => CallMethod("a", [0, 1])

    then the resulting dictionary will contain the entry
    `("a", CallMethod("a", [None, None]))`.
    """

    empties: typing.Dict[LenientNt, ReduceExprOrAccept] = {}

    def production_is_empty(p: Production) -> bool:
        return all(isinstance(e, LookaheadRule)
                   or isinstance(e, Optional)
                   or (isinstance(e, Nt) and e in empties)
                   or e is NoLineTerminatorHere
                   for e in p.body)

    def evaluate_reducer_with_empty_matches(p: Production) -> ReduceExprOrAccept:
        # partial evaluation of p.reducer
        stack = [e for e in p.body if is_concrete_element(e)]

        Expr = typing.TypeVar("Expr", ReduceExpr, ReduceExprOrAccept)

        def eval(expr: Expr) -> Expr:
            if expr is None:
                return None
            elif isinstance(expr, Some):
                return Some(eval(expr.inner))
            elif isinstance(expr, CallMethod):
                return dataclasses.replace(
                    expr,
                    args=tuple(eval(arg_expr) for arg_expr in expr.args)
                )
            elif isinstance(expr, int):
                e = stack[expr]
                if isinstance(e, Optional):
                    return None
                else:
                    assert isinstance(e, Nt)
                    result = empties[e]
                    assert not isinstance(result, str)
                    return result
            elif expr == 'accept':
                # Hmm, this is not ideal! Maybe 'accept' needs to take an
                # argument so that the normal case is Accept(0) and this case
                # is Accept(eval(expr.args[0])).
                return 'accept'
            else:
                raise TypeError(
                    "internal error: unhandled reduce expression type {!r}"
                    .format(expr))

        return eval(p.reducer)

    done = False
    while not done:
        done = True
        for nt, nt_def in grammar.nonterminals.items():
            if nt not in empties:
                for p in nt_def.rhs_list:
                    if production_is_empty(p):
                        if nt in empties:
                            raise ValueError(
                                "ambiguous grammar: multiple productions for "
                                "{!r} match the empty string"
                                .format(nt))
                        done = False
                        empties[nt] = evaluate_reducer_with_empty_matches(p)
    return empties

def check_cycle_free(grammar: Grammar) -> None:
    """Throw an exception if any nonterminal in `grammar` produces itself
    via a cycle of 1 or more productions.
    """
    empties = empty_nt_set(grammar)

    # OK, first find out which nonterminals directly produce which other
    # nonterminals (after possibly erasing some optional/empty nts).
    direct_produces: typing.Dict[LenientNt, typing.Set[Nt]] = {}
    for orig in grammar.nonterminals:
        direct_produces[orig] = set()
        for source_production in grammar.nonterminals[orig].rhs_list:
            for rhs, _r in expand_optional_symbols_in_rhs(source_production.body, grammar, empties):
                result: typing.List[Nt] = []
                all_possibly_empty_so_far = True
                # If we break out of the following loop, that means it turns
                # out that this production does not produce *any* strings that
                # are just a single nonterminal.
                for e in rhs:
                    if grammar.is_terminal(e):
                        break  # no good, this production contains a terminal
                    elif isinstance(e, Nt):
                        if e in empties:
                            if all_possibly_empty_so_far:
                                result.append(e)
                        else:
                            if not all_possibly_empty_so_far:
                                # Give up - we have 2+ nonterminals that can't
                                # be empty.
                                break
                            all_possibly_empty_so_far = False
                            result = [e]
                    elif isinstance(e, Exclude):
                        if isinstance(e.inner, Nt):
                            result.append(e.inner)
                    elif isinstance(e, LookaheadRule):
                        # Ignore the restriction. We lose a little precision
                        # here, and could report a cycle where there isn't one,
                        # but it's unlikely in real-world grammars.
                        pass
                    elif e is NoLineTerminatorHere:
                        # This doesn't affect the property we're checking.
                        pass
                    elif isinstance(e, Literal):
                        if e.text != "":
                            # This production contains a non-empty character,
                            # therefore it cannot correspond to an empty cycle.
                            break
                    elif isinstance(e, UnicodeCategory):
                        # This production consume a class of character,
                        # therefore it cannot correspond to an empty cycle.
                        break
                    elif isinstance(e, End):
                        # This production consume the End meta-character,
                        # therefore it cannot correspond to an empty cycle,
                        # even if this character is expect to be produced
                        # infinitely once the end is reached.
                        break
                    elif isinstance(e, CallMethod):
                        # This production execute code, but does not consume
                        # any input.
                        pass
                    else:
                        # Optional is not possible because we called
                        # expand_optional_symbols_in_rhs. ErrorSymbol
                        # effectively matches the empty string (though only if
                        # nothing else matches).
                        assert isinstance(e, ErrorSymbol)
                else:
                    # If we get here, we didn't break, so our results are good!
                    # nt can definitely produce all the nonterminals in result.
                    direct_produces[orig] |= set(result)

    def step(
            produces: typing.Dict[LenientNt, typing.Set[Nt]]
    ) -> typing.Dict[LenientNt, typing.Set[Nt]]:
        return {
            orig: dest | set(b for a in dest for b in produces[a])
            for orig, dest in produces.items()
        }
    produces = fix(step, direct_produces)

    for nt in grammar.nonterminals:
        if nt in produces[nt]:
            raise ValueError(
                "invalid grammar: nonterminal {} can produce itself"
                .format(nt))

def check_lookahead_rules(grammar: Grammar) -> None:
    """Check that no LookaheadRule appears at the end of a production (or before
    elements that can produce the empty string).

    If there are any offending lookahead rules, throw a ValueError.
    """

    empties = empty_nt_set(grammar)

    check_cycle_free(grammar)
    for nt in grammar.nonterminals:
        for source_production in grammar.nonterminals[nt].rhs_list:
            body = source_production.body
            for rhs, _r in expand_optional_symbols_in_rhs(body, grammar, empties):
                # XXX BUG: The next if-condition is insufficient, since it
                # fails to detect a lookahead restriction followed by a
                # nonterminal that can match the empty string.
                if rhs and isinstance(rhs[-1], LookaheadRule):
                    raise ValueError(
                        "invalid grammar: lookahead restriction "
                        "at end of production: {}"
                        .format(grammar.production_to_str(nt, body)))

def check_no_line_terminator_here(grammar: Grammar) -> None:
    empties = empty_nt_set(grammar)

    def check(e: Element, nt: LenientNt, body: typing.List[Element]) -> None:
        if grammar.is_terminal(e):
            pass
        elif isinstance(e, Nt):
            if e in empties:
                raise ValueError(
                    "invalid grammar: [no LineTerminator here] cannot appear next to "
                    "a nonterminal that matches the empty string\n"
                    "in production: {}".format(grammar.production_to_str(nt, body)))
        else:
            raise ValueError(
                "invalid grammar: [no LineTerminator here] must appear only "
                "between terminals and/or nonterminals\n"
                "in production: {}".format(grammar.production_to_str(nt, body)))

    for nt in grammar.nonterminals:
        for production in grammar.nonterminals[nt].rhs_list:
            body = production.body
            for i, e in enumerate(body):
                if e is NoLineTerminatorHere:
                    if i == 0 or i == len(body) - 1:
                        raise ValueError(
                            "invalid grammar: [no LineTerminator here] must be between two symbols\n"
                            "in production: {}".format(grammar.production_to_str(nt, body)))
                    check(body[i - 1], nt, body)
                    check(body[i + 1], nt, body)

def expand_parameterized_nonterminals(grammar: Grammar) -> Grammar:
    """Replace parameterized nonterminals with specialized copies.

    For example, a single pair `nt_name: NtDef(params=('A', 'B'), ...)` in
    `grammar.nonterminals` will be replaced with (assuming A and B are boolean
    parameters) up to four pairs, each having an Nt object as the key and an
    NtDef with no parameters as the value.

    `grammar.nonterminals` must have string keys.

    Returns a new copy of `grammar` with Nt keys, whose NtDefs all have
    `nt_def.params == []`.
    """

    todo = collections.deque(grammar.goals())
    new_nonterminals = {}

    def expand(nt: Nt) -> NtDef:
        """Expand grammar.nonterminals[nt](**args).

        Returns the expanded NtDef, which contains no conditional
        productions or Nt objects.
        """

        if nt.args is None:
            args_dict = None
        else:
            args_dict = dict(nt.args)

        def evaluate_arg(arg: NtParameter) -> NtParameter:
            if isinstance(arg, Var):
                return args_dict[arg.name]
            else:
                return arg

        def expand_element(e: Element) -> Element:
            if isinstance(e, Optional):
                return Optional(expand_element(e.inner))
            elif isinstance(e, Exclude):
                return Exclude(expand_element(e.inner), tuple(map(expand_element, e.exclusion_list)))
            elif isinstance(e, Nt):
                args = tuple((name, evaluate_arg(arg))
                             for name, arg in e.args)
                e = Nt(e.name, args)
                if e not in new_nonterminals:
                    todo.append(e)
                return e
            else:
                return e

        def expand_production(p: Production) -> Production:
            return p.copy_with(
                body=[expand_element(e) for e in p.body],
                condition=None)

        def expand_productions(nt_def: NtDef) -> NtDef:
            result = []
            for p in nt_def.rhs_list:
                if p.condition is None:
                    included = True
                else:
                    param, value = p.condition
                    included = (args_dict[param] == value)
                if included:
                    result.append(expand_production(p))
            return NtDef((), result, nt_def.type)

        nt_def = grammar.nonterminals[nt.name]
        assert tuple(name for name, value in nt.args) == nt_def.params
        return expand_productions(nt_def)

    while todo:
        nt = todo.popleft()
        if nt not in new_nonterminals:  # not already expanded
            new_nonterminals[nt] = expand(nt)

    # "type: ignore" because this runs afoul of Python's covariance rules for
    # Mapping; but it conforms to the intended type.
    return grammar.with_nonterminals(new_nonterminals)  # type: ignore

# *** Start sets and follow sets **********************************************

EMPTY = "(empty)"
END = None

TerminalOrEmpty = str
TerminalOrEmptyOrErrorToken = typing.Union[str, ErrorTokenClass]
StartSets = typing.Dict[Nt, OrderedFrozenSet[TerminalOrEmptyOrErrorToken]]

def start_sets(grammar: Grammar) -> StartSets:
    """Compute the start sets for nonterminals in a grammar.

    A nonterminal's start set is the set of tokens that a match for that
    nonterminal may start with, plus EMPTY if it can match the empty string
    and ErrorToken if it can start with an error.
    """

    # How this works: Note that we can replace the words "match" and "start
    # with" in the definition above with more queries about start sets.
    #
    # 1.  A nonterminal's start set contains a terminal `t` if any of its
    #     productions contains either `t` or a nonterminal with `t` in *its*
    #     start set, preceded only by zero or more nonterminals that have EMPTY
    #     in *their* start sets. Plus:
    #
    # 2.  A nonterminal's start set contains EMPTY if any of its productions
    #     consists entirely of nonterminals that have EMPTY in *their* start
    #     sets.
    #
    # This definition is rather circular. We want the smallest collection of
    # start sets satisfying these rules, and we get that by iterating to a
    # fixed point.

    assert all(isinstance(nt, Nt) for nt in grammar.nonterminals)
    start: StartSets
    start = {typing.cast(Nt, nt): OrderedFrozenSet() for nt in grammar.nonterminals}
    done = False
    while not done:
        done = True
        for nt, nt_def in grammar.nonterminals.items():
            assert isinstance(nt, Nt)
            # Compute start set for each `prod` based on `start` so far.
            # Could be incomplete, but we'll ratchet up as we iterate.
            nt_start = OrderedFrozenSet(
                t for p in nt_def.rhs_list for t in seq_start(grammar, start, p.body))
            if nt_start != start[nt]:
                start[nt] = nt_start
                done = False
    return start

def seq_start(
        grammar: Grammar,
        start: StartSets,
        seq: typing.List[Element]
) -> OrderedFrozenSet[TerminalOrEmptyOrErrorToken]:
    """Compute the start set for a sequence of elements."""
    s: OrderedSet[TerminalOrEmptyOrErrorToken] = OrderedSet([EMPTY])
    for i, e in enumerate(seq):
        if EMPTY not in s:  # preceding elements never match the empty string
            break
        s.remove(EMPTY)
        if grammar.is_terminal(e):
            assert isinstance(e, str)
            s.add(e)
        elif isinstance(e, ErrorSymbol):
            s.add(ErrorToken)
        elif isinstance(e, Nt):
            s |= start[e]
        elif e is NoLineTerminatorHere:
            s.add(EMPTY)
        else:
            assert isinstance(e, LookaheadRule)
            future = seq_start(grammar, start, seq[i + 1:])
            if e.positive:
                future &= e.set
            else:
                future -= e.set
            return OrderedFrozenSet(future)
    return OrderedFrozenSet(s)

StartSetCache = typing.List[typing.List[OrderedFrozenSet[TerminalOrEmptyOrErrorToken]]]

def make_start_set_cache(
        grammar: Grammar,
        prods: typing.List[Prod],
        start: StartSets
) -> StartSetCache:
    """Compute start sets for all suffixes of productions in the grammar.

    Returns a list of lists `cache` such that
    `cache[n][i] == seq_start(grammar, start, prods[n][i:])`.

    (The cache is for speed, since seq_start was being called millions of
    times.)
    """

    def suffix_start_list(
            rhs: typing.List[Element]
    ) -> typing.List[OrderedFrozenSet[TerminalOrEmptyOrErrorToken]]:
        sets: typing.List[OrderedFrozenSet[TerminalOrEmptyOrErrorToken]]
        sets = [OrderedFrozenSet([EMPTY])]
        for e in reversed(rhs):
            s: OrderedFrozenSet[TerminalOrEmptyOrErrorToken]
            if grammar.is_terminal(e):
                assert isinstance(e, str)
                s = OrderedFrozenSet([e])
            elif isinstance(e, ErrorSymbol):
                s = OrderedFrozenSet([ErrorToken])
            elif isinstance(e, Nt):
                s = start[e]
                if EMPTY in s:
                    s = OrderedFrozenSet((s - {EMPTY}) | sets[-1])
            elif e is NoLineTerminatorHere:
                s = sets[-1]
            else:
                assert isinstance(e, LookaheadRule)
                if e.positive:
                    s = OrderedFrozenSet(sets[-1] & e.set)
                else:
                    s = OrderedFrozenSet(sets[-1] - e.set)
            assert isinstance(s, OrderedFrozenSet)
            assert s == seq_start(grammar, start, rhs[len(rhs) - len(sets):])
            sets.append(s)
        sets.reverse()
        assert sets == [seq_start(grammar, start, rhs[i:])
                        for i in range(len(rhs) + 1)]
        return sets

    return [suffix_start_list(prod.rhs) for prod in prods]

FollowSet = OrderedSet[typing.Union[TerminalOrEmptyOrErrorToken, None]]
FollowSets = typing.DefaultDict[Nt, FollowSet]

def follow_sets(
        grammar: Grammar,
        prods_with_indexes_by_nt: typing.DefaultDict[
            LenientNt,
            typing.List[typing.Tuple[int, typing.List[Element]]]
        ],
        start_set_cache: StartSetCache
) -> FollowSets:
    """Compute all follow sets for nonterminals in a grammar.

    The follow set for a nonterminal `A`, as defined in the book, is "the set
    of terminals that can appear immediately to the right of `A` in some
    sentential form"; plus, "If `A` can be the rightmost symbol in some
    sentential form, then $ is in FOLLOW(A)."

    Returns a default-dictionary mapping nts to follow sets.
    """

    # Set of nonterminals already seen, including those we are in the middle of
    # analyzing. The algorithm starts at `goal` and walks all reachable
    # nonterminals, recursively.
    visited = set()

    # The results. By definition, nonterminals that are not reachable from the
    # goal nt have empty follow sets.
    follow: FollowSets = collections.defaultdict(OrderedSet)

    # If `(x, y) in subsumes_relation`, then x can appear at the end of a
    # production of y, and therefore follow[x] should be <= follow[y].
    # (We could maintain that invariant throughout, but at present we
    # brute-force iterate to a fixed point at the end.)
    subsumes_relation: OrderedSet[typing.Tuple[Nt, Nt]]
    subsumes_relation = OrderedSet()

    # `END` is $. It is, of course, in follow[each goal nonterminal]. It gets
    # into other nonterminals' follow sets through the subsumes relation.
    for init_nt in grammar.init_nts:
        assert isinstance(init_nt, Nt)
        follow[init_nt].add(END)

    def visit(nt: Nt) -> None:
        if nt in visited:
            return
        visited.add(nt)
        for prod_index, rhs in prods_with_indexes_by_nt[nt]:
            for i, symbol in enumerate(rhs):
                if isinstance(symbol, Nt):
                    visit(symbol)
                    after = start_set_cache[prod_index][i + 1]
                    if EMPTY in after:
                        after -= {EMPTY}
                        subsumes_relation.add((symbol, nt))
                    follow[symbol] |= after

    for nt in grammar.init_nts:
        assert isinstance(nt, Nt)
        visit(nt)

    # Now iterate to a fixed point on the subsumes relation.
    done = False
    while not done:
        done = True  # optimistically
        for target, source in subsumes_relation:
            if follow[source] - follow[target]:
                follow[target] |= follow[source]
                done = False

    return follow

# *** Lowering ****************************************************************

# At this point, lowered productions start getting farther from the original
# source.  We need to associate them with the original grammar in order to
# produce correct output, so we use Prod values to represent productions.
#
# -   `nt` is the name of the nonterminal as it appears in the original
#     grammar.
#
# -   `index` is the index of the source production, within nt's productions,
#     in the original grammar.
#
# -   `rhs` is the fully lowered/expanded right-hand-side of the production.
#
# There may be many productions in a grammar that all have the same `nt` and
# `index` because they were all produced from the same source production.
@dataclass
class Prod:
    nt: Nt
    index: int
    rhs: typing.List
    reducer: ReduceExprOrAccept

def expand_optional_symbols_in_rhs(
        rhs: typing.List[Element],
        grammar: Grammar,
        empties: typing.Dict[LenientNt, ReduceExprOrAccept],
        start_index: int = 0
) -> typing.Iterable[typing.Tuple[typing.List[Element], typing.Dict[int, ReduceExpr]]]:
    """Expand a sequence with optional symbols into sequences that have none.

    rhs is a list of symbols, possibly containing optional elements. This
    yields every list that can be made by replacing each optional element
    either with its .inner value, or with nothing.

    Each list is accompanied by the list of the indices of optional elements in
    `rhs` that were dropped.

    For example, `expand_optional_symbols_in_rhs(["if", Optional("else")])`
    yields the two pairs `(["if"], [1])` and `["if", "else"], []`.
    """

    replacement: ReduceExpr
    for i in range(start_index, len(rhs)):
        e = rhs[i]
        if isinstance(e, Optional):
            if isinstance(e.inner, Nt) and e.inner in empties:
                # If this is already possibly-empty in the input grammar, it's an
                # error! The grammar is ambiguous.
                raise ValueError(
                    "ambiguous grammar: {} is ambiguous because {} can match "
                    "the empty string"
                    .format(grammar.element_to_str(e),
                            grammar.element_to_str(e.inner)))
            replacement = None
            break
        elif isinstance(e, Nt) and e in empties:
            empty_expr = empties[e]
            # The replacement can't be 'accept' because that only happens with
            # InitNt nonterminals, which are never used in productions.
            assert not isinstance(empty_expr, str)
            replacement = empty_expr
            break
    else:
        yield rhs[start_index:], {}
        return

    for expanded, r in expand_optional_symbols_in_rhs(rhs, grammar, empties, i + 1):
        e = rhs[i]
        rhs_inner = e.inner if isinstance(e, Optional) else e
        # without rhs[i]
        r2 = r.copy()
        r2[i] = replacement
        yield rhs[start_index:i] + expanded, r2
        # with rhs[i]
        yield rhs[start_index:i] + [rhs_inner] + expanded, r

def expand_all_optional_elements(grammar: Grammar) -> typing.Tuple[
        Grammar,
        typing.List[Prod],
        typing.DefaultDict[LenientNt, typing.List[typing.Tuple[int, typing.List[Element]]]]
]:
    """Expand optional elements in the grammar.

    We replace each production that contains an optional element with two
    productions: one with and one without. Downstream of this step, we can
    ignore the possibility of optional elements.
    """
    expanded_grammar: typing.Dict[LenientNt, NtDef] = {}

    # This was capturing the set of empty production to simplify the work of
    # the previous algorithm which was trying to determine the lookahead.
    # However, with the LR0Generator this is no longer needed as we are
    # generating deliberatly inconsistent parse table states, which are then
    # properly fixed by adding lookahead information where needed, and without
    # bugs!
    # empties = empty_nt_set(grammar)
    empties: typing.Dict[LenientNt, ReduceExprOrAccept] = {}

    # Put all the productions in one big list, so each one has an index. We
    # will use the indices in the action table (as reduce action payloads).
    prods: typing.List[Prod] = []
    prods_with_indexes_by_nt: \
        typing.DefaultDict[LenientNt, typing.List[typing.Tuple[int, typing.List[Element]]]] = \
        collections.defaultdict(list)

    for nt, nt_def in grammar.nonterminals.items():
        assert isinstance(nt, Nt)
        prods_expanded = []
        for prod_index, p in enumerate(nt_def.rhs_list):
            # Aggravatingly, a reduce-expression that's an int is not
            # simply an offset into p.body. It only counts "concrete"
            # elements. Make a mapping for converting reduce-expressions to
            # offsets.
            reduce_expr_to_offset = [
                i
                for i, e in enumerate(p.body)
                if is_concrete_element(e)
            ]

            for pair in expand_optional_symbols_in_rhs(p.body, grammar, empties):
                expanded_rhs, removals = pair

                Expr = typing.TypeVar("Expr", ReduceExpr, ReduceExprOrAccept)

                def adjust_reduce_expr(expr: Expr) -> Expr:
                    if isinstance(expr, int):
                        i = reduce_expr_to_offset[expr]
                        if i in removals:
                            return removals[i]
                        was_optional = isinstance(p.body[i], Optional)
                        expr -= sum(1 for r in removals if r < i)
                        if was_optional:
                            return Some(expr)
                        else:
                            return expr
                    elif expr is None:
                        return None
                    elif isinstance(expr, Some):
                        return Some(adjust_reduce_expr(expr.inner))
                    elif isinstance(expr, CallMethod):
                        return dataclasses.replace(
                            expr,
                            args=tuple(adjust_reduce_expr(arg)
                                       for arg in expr.args)
                        )
                    elif expr == 'accept':
                        # doesn't need to be adjusted because 'accept' isn't
                        # turned into code downstream.
                        return 'accept'
                    else:
                        raise TypeError(
                            "internal error: unrecognized element {!r}"
                            .format(expr))

                adjusted_reducer = adjust_reduce_expr(p.reducer)
                prods_expanded.append(
                    Production(body=expanded_rhs,
                               reducer=adjusted_reducer))
                prods.append(Prod(nt, prod_index, expanded_rhs,
                                  adjusted_reducer))
                prods_with_indexes_by_nt[nt].append(
                    (len(prods) - 1, expanded_rhs))
        expanded_grammar[nt] = nt_def.with_rhs_list(prods_expanded)

    return (grammar.with_nonterminals(expanded_grammar),
            prods,
            prods_with_indexes_by_nt)

class CanonicalGrammar:
    __slots__ = ["prods", "prods_with_indexes_by_nt", "grammar"]

    prods: typing.List[Prod]
    prods_with_indexes_by_nt: typing.Mapping[
        LenientNt,
        typing.List[typing.Tuple[int, typing.List[Element]]]]
    grammar: Grammar

    def __init__(self, grammar: Grammar) -> None:
        # Step by step, we check the grammar and lower it to a more primitive form.
        grammar = expand_parameterized_nonterminals(grammar)
        check_cycle_free(grammar)
        # check_lookahead_rules(grammar)
        check_no_line_terminator_here(grammar)
        grammar, prods, prods_with_indexes_by_nt = \
            expand_all_optional_elements(grammar)

        self.prods = prods
        self.prods_with_indexes_by_nt = prods_with_indexes_by_nt
        self.grammar = grammar


"""
This is a fairly basic scraper built from BeautifulSoup4 designed to
pull problem pages from projecteuler.net and build a markdown file
from them. Naturally this is not a very general script.

Some omissions (for the sake of simplicity) include:
- Does not bother with bold or italics or other similar formattings
- Uses characters '^' and '_' to denote super and subscripts respectively
- Utilises a bare minimum of formatting for tables

This script being for my personal use it also organises the .py and
.txt files associated with a given problem. It creates a subdirectory
for each problem, copies the .py file if there is a comment containing
'CORRECT', removes the answer from the comment, and strips the problem
description from the .py file as the README.md file will contain it
instead. (There are option flags for overriding files and filtering
the answer and description.)

Problem .py files are stipped of their answers out of respect for the
owners/operators of Project Euler, as they request users do not
publish answers.

A delay of 3sec is implemented between each problem it handles if
a markdown file was successfully created. This is to prevent spamming
of the Project Euler site as a courtesy.

"""

import sys
sys.path.insert(0,"/home/dco/virtenvBS4/lib/python3.5/site-packages")
import os
import shutil
import fnmatch

import requests
import time
from bs4 import BeautifulSoup
from bs4 import NavigableString

class HTMLParser:
    working_dir = "ProjectEulerSolutions/"
    base_URL = "https://projecteuler.net/"
    post_data = "problem"
    root_path = "/home/dco/PythonProjects/"

    def __init__(self, override_py = False, override_md = False, filter_ans = True, filter_desc = True):
        self.override_python = override_py
        self.override_markdown = override_md
        self.filter_answers = filter_ans
        self.filter_description = filter_desc
        self.PEproblem_list = self.findFile('PE*.py', self.root_path)
        self.PEproblem_list = ['PE45-TriangularPentagonalHexagonal.py']

    # File helper functions
    def findFile(self, pattern, path):
        result = []
        for file in os.listdir(path):
            if fnmatch.fnmatch(file, pattern):
                result.append(file)
        return result

    def readFileLine(self, file):
        with open(file) as file:
            for line in file:
                yield(line)

    def checkProblemCompletion(self, source_file):
        with open(source_file) as file:
            for line in file:
                if line.find("CORRECT") != -1:
                    return True
        return False

    def copyPEPyFile(self, source_file, destination_file):
        header = "PROJECT EULER PROBLEM"
        with open(destination_file, 'w') as destination:
            source_lines = readFileLine(source_file)
            line = ''
            filtered_desc = False
            while True:
                try:
                    line = next(source_lines)
                    if line == '"""\n' and not filtered_desc and self.filter_description:
                        line = next(source_lines)
                        if line.find(header) != -1:
                            destination.write("# " + line)
                            line = next(source_lines)
                            while line != '"""\n':
                                line = next(source_lines)
                            filtered_desc = True
                    elif line.find("CORRECT") != -1 and self.filter_answers:
                        pass
                    else:
                        destination.write(line)
                except StopIteration:
                    break

    # URL helper functions
    def digits(self, str_x):
        out_s = ''
        for s in str_x:
            if s in "0123456789":
                out_s += s
        return(out_s)

    # Tag to Markdown parsing helper functions
    def parseP(self, content):
        self.parseLink(content)
        self.parseBR(content)
        self.parseSub(content)
        self.parseSuper(content)

    def parseLink(self, content):
        links = content.select('a')
        for link in links:
            href = str(link['href'])
            text = str(link.text)
            link.replace_with("["+text+"]("+href+")")

    def parseBR(self, content):
        BRs = content.select('br')
        for BR in BRs:
            BR.replace_with("\n")

    def parseSub(self, content):
        subscripts = content.select('sub')
        for subscript in subscripts:
            text = subscript.text
            subscript.replace_with("_" + text)

    def parseSuper(self, content):
        superscripts = content.select('sup')
        for superscript in superscripts:
            text = superscript.text
            superscript.replace_with("^" + text)

    def parseTable(self, content, lines):
        rows = content.select('tr')
        for row in rows:
            new_text = ""
            cells = row.select('td')
            for i,cell in enumerate(cells):
                text = cell.text
                if text and text.strip():
                    if i == 0:
                        new_text += text
                    else:
                        new_text += " | " + text
            new_text += "\n"
            row.replace_with(new_text)
            line = "\n" + str(row.text) + "\n"
            lines.append(line)

    def parseIMG(self, img_content):
        return('![]('+img_content+')')

    # Main execution script
    def execute(self):
        for problem_str in self.PEproblem_list:
            print("{} found in {}.".format(problem_str, self.root_path))
            problem_number = self.digits(problem_str.split('-')[0])
            source_file = self.root_path + problem_str

            if self.checkProblemCompletion(source_file):

                problem_URL = self.base_URL + self.post_data + '=' + problem_number
                sub_dir = "PE" + problem_number + "/"
                sub_path = self.root_path + self.working_dir + sub_dir
                python_filename = sub_path + problem_str

                # Make the appropriate subdir if it DNE
                try:
                    os.makedirs(sub_path)
                    print("Created subdirectory {} in {}".format(sub_dir, self.root_path))
                except OSError:
                    if not os.path.isdir(sub_path):
                        raise
                    else:
                        print("Subdirectory {} already exists in {}".format(sub_dir, self.root_path))

                markdown_filename = sub_path + "README.md"
                markdown_file_exists = os.path.isfile(markdown_filename)

                # Generate markdown
                if not markdown_file_exists or self.override_markdown:
                    print("Pulling data from {}...".format(self.base_URL))
                    r = requests.get(problem_URL)
                    soup = BeautifulSoup(r.content, "html.parser")
                    content = soup.find(id='content')
                    title = content.h2.text

                    body = content.find(class_='problem_content')

                    print("Generating markdown file...")
                    lines = ["# " + title + "\n", "## [Problem " + problem_number + "](" + problem_URL + ")\n"]
                    for tag in body.findChildren():
                        if tag.name == 'p':
                            self.parseP(tag)
                            line = str(tag.get_text()) + "\n"
                            lines.append(line)
                        elif tag.name == 'div':
                            img = tag.img
                            if img:
                                line = self.parseIMG(self.base_URL + tag.img['src']) + "\n"
                                lines.append(line)
                        elif tag.name == 'br':
                            lines.append("\n")
                        elif tag.name == 'table':
                            self.parseTable(tag, lines)

                    with open(markdown_filename,'w') as file:
                        for line in lines:
                            file.write(line)
                else:
                    print("README.md already exists in {}".format(sub_path))

                # Relocate .py file to project-problem directory
                python_file_exists = os.path.isfile(python_filename)
                if not python_file_exists or self.override_python:
                    try:
                        copyPEPyFile(source_file, python_filename)
                        print("Python file created: {}".format(python_filename))
                        print(" Filtered description: {}".format(filter_description))
                        print(" Filtered answer: {}".format(filter_answer))
                    except:
                        print("Error generating python file copy at {}!".format(python_filename))
                else:
                    print("Python file has already been copied to {}".format(sub_dir))

                # Prepare .txt files associated with the problem and copy it to the sub_dir
                leading0s = "".join(['0' for i in range(3 - len(problem_number))])
                text_filenames = self.findFile('p' + leading0s + problem_number + '*.txt', self.root_path)

                if len(text_filenames):
                    print("Found the following associated text files:")
                    for f in text_filenames:
                        print("\t{}".format(f))
                        if os.path.isfile(sub_path + f):
                            print("{} already exists in {}".format(f, sub_path))
                        else:
                            shutil.copy2(self.root_path + f, sub_path + f)
                            print("{} copied to {}".format(f, sub_path))

                # If a markdown file was created then wait 3sec before next webscrape
                if not markdown_file_exists:
                    time.sleep(3)

            else:
                print("Problem {} is not marked as a correct solution and has been skipped!".format(problem_number))

            print()

if __name__ == "__main__":
    parser = HTMLParser()
    parser.execute()


import tempfile
import unittest

from pyspark import Row

from sourced.ml.models import DocumentFrequencies
from sourced.ml.tests import create_spark_for_test
from sourced.ml.transformers import Indexer

class IndexerTests(unittest.TestCase):
    def setUp(self):
        data = [Row(to_index="to_index%d" % i, value=i) for i in range(10)]
        self.data = data
        self.session = create_spark_for_test()
        self.data_rdd = self.session.sparkContext \
            .parallelize(range(len(data))) \
            .map(lambda x: data[x])

    def test_call(self):
        indexer = Indexer("to_index")
        res = indexer(self.data_rdd)
        values = indexer.values()
        data_reverse = res \
            .map(lambda x: Row(to_index=values[x.to_index], value=x.value)) \
            .collect()
        self.assertEqual(self.data, data_reverse)

    def test_save_load(self):
        indexer = Indexer("to_index")
        res = indexer(self.data_rdd)
        with tempfile.NamedTemporaryFile(suffix="-index.asdf") as tmp:
            cached_index_path = tmp.name
            indexer.save_index(cached_index_path)
            docfreq = DocumentFrequencies().load(source=cached_index_path)
            document_index = {key: int(val) for (key, val) in docfreq}
            indexer = Indexer("to_index", column2id=document_index)
            self.assertEqual(res.collect(), indexer(self.data_rdd).collect())

if __name__ == "__main__":
    unittest.main()


import pandas as pd

# read the csv file
df = pd.read_csv(r'C:\Users\annel\Downloads\ITS66604 Machine Learning and Parallel Computing\labs\Lab9_RegressionPracticalMaterialsV1.03\weight-height.csv')

# first 5 rows of the data set
df.head()

# shape of the dataframe
df.shape

# data type of each column
df.dtypes

# number of null values
df.info()

# number of unique values of column Gender
df.Gender.nunique()
# 2

# unique values of column Gender
df.Gender.unique()
# array(['Male', 'Female'], dtype=object)

# import plots
import matplotlib.pyplot as plt

plt.style.use('ggplot')

# histogram of Height
df.Height.plot(kind='hist', color='purple', edgecolor='black', figsize=(10,7))
plt.title('Distribution of Height', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Frequency', size=18) # y axis label

# histogram of Weight
df.Weight.plot(kind='hist', color='purple', edgecolor='black', figsize=(10,7))
plt.title('Distribution of Weight', size=24) # title of plot
plt.xlabel('Weight (pounds)', size=18) # x axis label
plt.ylabel('Frequency', size=18) # y axis label

# histogram of Height - males and females
df[df['Gender'] == 'Male'].Height.plot(kind='hist', color='blue', edgecolor='black',
                                       alpha=0.5, figsize=(10, 7))
df[df['Gender'] == 'Female'].Height.plot(kind='hist', color='magenta',
                                         edgecolor='black', alpha=0.5,
                                         figsize=(10, 7))
plt.legend(labels=['Males', 'Females']) # legend
plt.title('Distribution of Height', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Frequency', size=18) # y axis label

# histogram of Weight - males and females
df[df['Gender'] == 'Male'].Weight.plot(kind='hist', color='blue', edgecolor='black',
                                       alpha=0.5, figsize=(10, 7))
df[df['Gender'] == 'Female'].Weight.plot(kind='hist', color='magenta',
                                         edgecolor='black', alpha=0.5,
                                         figsize=(10, 7))
plt.legend(labels=['Males', 'Females']) # legend
plt.title('Distribution of Weight', size=24) # title of plot
plt.xlabel('Weight (pounds)', size=18) # x axis label
plt.ylabel('Frequency', size=18) # y axis label

# Descriptive statistics for male
statistics_male = df[df['Gender'] == 'Male'].describe()
statistics_male.rename(columns=lambda x: x + '_male', inplace=True)

# Descriptive statistics for female
statistics_female = df[df['Gender'] == 'Female'].describe()
statistics_female.rename(columns=lambda x: x + '_female', inplace=True)

# Dataframe that contains statistics for both male and female
statistics = pd.concat([statistics_male, statistics_female], axis=1)
statistics

# Scatter plot of Height and Weight
ax1 = df[df['Gender'] == 'Male'].plot(kind='scatter', x='Height', y='Weight',
                                      color='blue', alpha=0.5, figsize=(10, 7))
df[df['Gender'] == 'Female'].plot(kind='scatter', x='Height', y='Weight',
                                  color='magenta', alpha=0.5, figsize=(10 ,7),
                                  ax=ax1)
plt.legend(labels=['Males', 'Females']) # legend
plt.title('Relationship between Height and Weight', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18); # y axis label

# Scatter plot of 500 females
sample_females = df[df['Gender'] == 'Female'].sample(500) # 500 data points
sample_females.plot(kind='scatter', x='Height', y='Weight', color='magenta',
                    alpha=0.5, figsize=(10, 7))
plt.legend(labels=['Females']) # legend
plt.title('Relationship between Height and Weight (sample of 500 females)',
          size=20) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18); # y axis label

# import numpy for working with numbers
import numpy as np

# best fit polynomials
df_males = df[df['Gender'] == 'Male']
df_females = df[df['Gender'] == 'Female']

# polynomial - males
male_fit = np.polyfit(df_males.Height, df_males.Weight, 1)
male_fit
# array([ 5.96177381, -224.49884071])

# polynomial - females
female_fit = np.polyfit(df_females.Height, df_females.Weight, 1)
female_fit
# array([ 5.99404661, -246.01326575])

# scatter plots and regression lines
# males and females dataframes
df_males = df[df['Gender'] == 'Male']
df_females = df[df['Gender'] == 'Female']

# Scatter plots
ax1 = df_males.plot(kind='scatter', x='Height', y='Weight', color='blue',
                    alpha=0.5, figsize=(10, 7))
df_females.plot(kind='scatter', x='Height', y='Weight', color='magenta',
                alpha=0.5, figsize=(10, 7), ax=ax1)

# regression lines
plt.plot(df_males.Height, male_fit[0] * df_males.Height + male_fit[1],
         color='darkblue', linewidth=2)
plt.plot(df_females.Height, female_fit[0] * df_females.Height + female_fit[1],
         color='deeppink', linewidth=2)

# regression equations
plt.text(65, 230, 'y={:.2f}+{:.2f}*x'.format(male_fit[1], male_fit[0]),
         color='darkblue', size=12)
plt.text(70, 130, 'y={:.2f}+{:.2f}*x'.format(female_fit[1], female_fit[0]),
         color='deeppink', size=12)

# legend, title and labels
plt.legend(labels=['Males Regresion Line', 'Females Regresion Line',
                   'Males', 'Females']) # legend
plt.title('Relationship between Height and Weight', size=24) # title of plots
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18) # y axis label

# import searborn library for more variety of data visualisation using
# fewer syntax and interesting default themes
import seaborn as sns

# regression plot using seaborn
fig = plt.figure(figsize=(10, 7)) # fig size
sns.regplot(x=df_males.Height, y=df_males.Weight, color='blue', marker='+')
sns.regplot(x=df_females.Height, y=df_females.Weight, color='magenta', marker='+')

# Legend, title and labels.
plt.legend(labels=['Males', 'Females']) # legend
plt.title('Relationship between Height and Weight', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18) # y axis label

# 300 random samples for Male and Female
df_males_sample = df[df['Gender'] == 'Male'].sample(300)
df_females_sample = df[df['Gender'] == 'Female'].sample(300)

# regression plot using seabornfig = plt.figure(figsize=(10, 7))
sns.regplot(x=df_males_sample.Height, y=df_males_sample.Weight, color='blue',
            marker='+')
sns.regplot(x=df_females_sample.Height, y=df_females_sample.Weight,
            color='magenta', marker='+')
plt.legend(labels=['Males', 'Females']) # legend
plt.title('Relationship between Height and Weight', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18) # y axis label

# import linear regression model
from sklearn.linear_model import LinearRegression

# dataframe of Males
df_males = df[df['Gender'] == 'Male']

# create linear regression object
lr_males = LinearRegression()

# fit linear regression
lr_males.fit(df_males[['Height']], df_males['Weight'])

# intercept of the best fit line
print(lr_males.intercept_)
# -224.49884070545772

# slope of the best fit line
print(lr_males.coef_)
# 5.96177381

# dataframe of Female
df_females = df[df['Gender'] == 'Female']

# create linear regression object
lr_females = LinearRegression()

# fit linear regression
lr_females.fit(df_females[['Height']], df_females['Weight'])

# intercept of the best fit line
print(lr_females.intercept_)
# -246.01326574667277

# slope of the best fit line
print(lr_females.coef_)
# 5.99404661

# dataframe of Female
df_females = df[df['Gender'] == 'Female']

# fit the model using numpy
female_fit = np.polyfit(df_females.Height, df_females.Weight, 1)
female_fit

# predictions using numpy
print(np.polyval(female_fit, [60]))
# [113.62953114]

# fit the model using scikit learnlr_females = LinearRegression()
lr_females.fit(df_females[['Height']], df_females['Weight'])

# predictions using scikit learn
print(lr_females.predict([[60]]))
# [113.62953114]

# dataframe of Female
df_females = df[df['Gender'] == 'Female']

# correlation coefficients
df_females.corr()

# dataframe of Male
df_males = df[df['Gender'] == 'Male']

# correlation coefficients
df_males.corr()

from scipy import stats

# dataframe of Female
df_females = df[df['Gender'] == 'Female']

# pearson correlation coefficient and p-value
pearson_coef, p_value = stats.pearsonr(df_females.Height, df_females.Weight)
print(pearson_coef)
# 0.849608591418601

# dataframe of Male
df_males = df[df['Gender'] == 'Male']

# pearson correlation coefficient and p-value
pearson_coef, p_value = stats.pearsonr(df_males.Height, df_males.Weight)
print(pearson_coef)
# 0.8629788486163176

# dataframe of 500 data points of Female
df_females = df[df['Gender'] == 'Female'].sample(500)

# residual plot 500 females
fig = plt.figure(figsize = (10, 7)) # fig size
sns.residplot(df_females.Height, df_females.Weight, color='magenta')
plt.title('Residual plot 500 females', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18) # y axis label

# dataframe of 500 data points of Males
df_males = df[df['Gender'] == 'Male'].sample(500)

# residual plot of 500 males
fig = plt.figure(figsize=(10, 7)) # fig size
sns.residplot(df_males.Height, df_males.Weight, color='blue')
plt.title('Residual plot 500 males', size=24) # title of plot
plt.xlabel('Height (inches)', size=18) # x axis label
plt.ylabel('Weight (pounds)', size=18) # y axis label

# print(df)
df_dummy = pd.get_dummies(df)
print(df_dummy)
# df_dummy = pd.get_dummies(df, dummy_na=True)

# drop female column
df_dummy.drop('Gender_Female', axis=1, inplace=True)

# rename Gender_Male column
df_dummy.rename(columns={'Gender_Male': 'Gender'}, inplace=True)

# df_dummy dataframe first 5 columns
df_dummy.head()

# create linear regression object
mlr = LinearRegression()

# fit linear regression
mlr.fit(df_dummy[['Height', 'Gender']], df_dummy['Weight'])

# intercept of the best fit line
print(mlr.intercept_)
# -244.92350252069903

# slope of the best fit line
print(mlr.coef_)
# [ 5.97694123 19.37771052

lab8.3 - Simple and Multiple Linear Regression.py


# -*- coding: utf-8 -*-
# Режимы доступа (инкапуляция)
# Атрибут без подчеркивания, значит доступ публичный
# одно подчеркивание protected Одно нижнее подчеркивание только сигнализирует, но не мешает использовать
# два подчеркивания перед именем private
#
class Point:
    def __init__(self, x=0, y=0):
        self.__x = self.__y = 0
        if self.__check_value(x) and self.__check_value(y):
            self.__x = x
            self.__y = y

    @classmethod
    def __check_value(cls, x):  # Приватный метод
        return type(x) in (int, float)

    def set_coord(self, x, y):
        "Сеттер"
        if self.__check_value(x) and self.__check_value(y):
            self.__x = x
            self.__y = y
        else:
            raise ValueError("Кординаты должны быть числами")

    def get_coord(self):
        "Геттер"
        return self.__x, self.__y

pt = Point(1, 2)
pt.set_coord(100, 200)
print(pt.get_coord())

# Можно использовать библиотеку accessify чтобы сделать атрибуты класса приватными
# from accessify import private, protected Более сильная защита, чем два подчеркивания.


import sys
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import (
    QApplication, QWidget, QDesktopWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLineEdit, QMessageBox, QTableWidget,
    QTableWidgetItem, QLabel
)

from utils.database import DB

class MainWindow(QWidget):
    STATUS_MAPPING = {
        0: "初始化中",
        1: "待执行",
        2: "正在执行",
        3: "完成并提醒",
        10: "异常并停止",
        11: "初始化失败",
    }

    def __init__(self):
        super().__init__()
        self.init_ui()

    def init_ui(self):
        """ 将窗体和窗体中的元素给生成 """

        self.setWindowTitle('亚马逊检测平台')
        self.resize(1228, 550)

        # 窗体居中显示
        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)

        # 1. 垂直方向布局（元素垂直存放）
        layout = QVBoxLayout()

        # 1.1 顶部布局，里面有2个按钮
        layout.addLayout(self.init_header())

        # 1.2 输入框
        layout.addLayout(self.init_form())

        # 1.3 表格
        layout.addLayout(self.init_table())

        # 1.4 底部
        layout.addLayout(self.init_footer())

        # layout.addStretch(1)
        self.setLayout(layout)
        self.show()

    def init_header(self):
        header = QHBoxLayout()

        btn_start = QPushButton("开始")
        btn_start.clicked.connect(self.event_start_click)

        header.addWidget(btn_start)

        btn_stop = QPushButton("结束")
        btn_stop.clicked.connect(self.event_stop_click)
        header.addWidget(btn_stop)

        header.addStretch(1)
        return header

    def init_form(self):
        form = QHBoxLayout()

        self.txt_asin = txt_asin = QLineEdit()
        txt_asin.setPlaceholderText("请输入商品ASIN且多个用逗号分隔，如：B0815JJQQ8=18,B0815JJQQ9=19,B0818JJQQ8=88")

        form.addWidget(txt_asin)
        btn_add = QPushButton("添加")
        btn_add.clicked.connect(self.event_add_click)

        form.addWidget(btn_add)
        return form

    def init_table(self):
        table = QHBoxLayout()

        # 创建表格对象
        self.table_widget = table_widget = QTableWidget(0, 8)

        # 生成表头数据
        table_header_list = [
            {"field": "asin", "text": "ASIN", 'width': 120},
            {"field": "title", "text": "标题", 'width': 150},
            {"field": "url", "text": "URL", 'width': 400},
            {"field": "price", "text": "底价", 'width': 100},
            {"field": "success", "text": "成功次数", 'width': 100},
            {"field": "error", "text": "503次数", 'width': 100},
            {"field": "status", "text": "状态", 'width': 100},
            {"field": "frequency", "text": "频率（N秒/次）", 'width': 100},
        ]

        for index, info in enumerate(table_header_list):
            # 设置表格宽度
            table_widget.setColumnWidth(index, info['width'])
            item = QTableWidgetItem()
            item.setText(info['text'])
            table_widget.setHorizontalHeaderItem(index, item)

        # 创建表内容，在创建某些对象添加到表格中。 去文件中获取历史的商品信息
        db_data_list = DB.CACHE_LIST

        current_row_count = table_widget.rowCount()
        for item in db_data_list:
            # 原来的基础上再增加一行（空白）
            table_widget.insertRow(current_row_count)
            self.create_row(table_widget, item, current_row_count)
            current_row_count += 1

        table.addWidget(table_widget)
        return table

    def init_footer(self):
        footer = QHBoxLayout()

        self.label_status = label_status = QLabel("未检测", self)
        footer.addWidget(label_status)

        footer_config = QHBoxLayout()
        footer_config.addStretch(1)

        btn_reinit = QPushButton("重新初始化")
        footer_config.addWidget(btn_reinit, 0, Qt.AlignRight)
        # btn_reinit.clicked.connect(self.event_reinit_click)

        btn_recheck = QPushButton("重新检测")
        footer_config.addWidget(btn_recheck, 0, Qt.AlignRight)
        # btn_recheck.clicked.connect(self.event_recheck_click)

        btn_reset_count = QPushButton("次数清零")
        footer_config.addWidget(btn_reset_count, 0, Qt.AlignRight)
        # btn_reset_count.clicked.connect(self.event_reset_count_click)

        btn_delete = QPushButton("删除检测项")
        footer_config.addWidget(btn_delete, 0, Qt.AlignRight)
        # btn_delete.clicked.connect(self.event_delete_click)

        btn_alert = QPushButton("SMTP报警配置")
        footer_config.addWidget(btn_alert, 0, Qt.AlignRight)
        # btn_alert.clicked.connect(self.event_alert_click)

        btn_proxy = QPushButton("代理IP")
        footer_config.addWidget(btn_proxy, 0, Qt.AlignRight)
        # btn_proxy.clicked.connect(self.event_proxy_click)

        footer.addLayout(footer_config)
        return footer

    def create_row(self, table_widget, item, current_row_count):
        for column, ele in enumerate(item):
            # 规定，是否不可以被修改
            text = self.STATUS_MAPPING[item[column]] if column == 6 else item[column]
            cell = QTableWidgetItem(str(text))
            if column in [0, 4, 5, 6]:
                # 不能被修改
                cell.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
            table_widget.setItem(current_row_count, column, cell)

    def event_start_click(self):
        QMessageBox.warning(self, "错误", "点击开始")

    def event_stop_click(self):
        QMessageBox.warning(self, "错误", "点击结束")

    def event_add_click(self):
        """ 获取在输入框中输入的数据，然后再去处理 """
        text = self.txt_asin.text()
        if not text:
            QMessageBox.warning(self, "错误", "商品ASIN输入错误 ！")
            return

        text = text.replace("，", ",")
        asin_price_list = text.split(",")

        # 获取当前表格总共有多少行
        current_row_index = self.table_widget.rowCount()

        for item in asin_price_list:
            data_pair = item.split("=")
            if len(data_pair) != 2:
                continue
            try:
                asin, price = data_pair
                asin = asin.strip()
                price = float(price.strip())
            except Exception as e:
                QMessageBox.warning(self, "错误", "商品ASIN输入错误 ！")
                return

            # ASIN已存在，自动忽略不再添加
            if DB.get_by_asin(asin):
                continue

            # 表格添加 & 数据库插入（asin，价格，状态，次数）
            new_row_data_list = [asin, "", "", price, 0, 0, 0, 5]
            DB.add(new_row_data_list)

            self.table_widget.insertRow(current_row_index)
            self.create_row(self.table_widget, new_row_data_list, current_row_index)

            # 在创建一个线程去获取商品标题、
            from utils.thread import NewTaskThread
            thread = NewTaskThread(asin, self)
            thread.updated.connect(self.init_task_success_callback)
            thread.error.connect(self.init_task_error_callback)
            thread.start()

            current_row_index += 1

    def init_task_success_callback(self, asin, title, url):

        # 去文件中找
        db_index = DB.get_index_by_asin(asin)
        if db_index == None:
            return

        row_object = self.table_widget.item(db_index, 0)
        row_asin = row_object.text().strip()
        if row_asin != asin:
            return

        # 更新数据库中的：标题、URL、状态
        DB.update_title_url_status(asin, title, url, 1)

        # 更新表格中的：标题、URL、状态
        cell_title = QTableWidgetItem(title)
        self.table_widget.setItem(db_index, 1, cell_title)

        cell_url = QTableWidgetItem(url)
        self.table_widget.setItem(db_index, 2, cell_url)

        cell_status = QTableWidgetItem(self.STATUS_MAPPING[1])
        cell_status.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
        self.table_widget.setItem(db_index, 6, cell_status)

        # 如果已启动，则开始检测
        # if self.switch == SWITCH.RUNNING:
        #     SCHEDULER.start_one(
        #         db_index,
        #         asin,
        #         self,
        #         self.task_start_callback,
        #         self.task_stop_callback,
        #         self.task_counter_callback,
        #         self.task_error_counter_callback,
        #         self.task_error_callback,
        #         self.task_success_callback
        #     )
        self.txt_asin.clear()

    def init_task_error_callback(self, asin, title, detail):
        db_index = DB.get_index_by_asin(asin)
        if db_index == None:
            return
        row_object = self.table_widget.item(db_index, 0)
        if not row_object:
            return
        row_asin = row_object.text().strip()
        if row_asin != asin:
            return
        # 更新数据库状态：初始化失败 & 写入错误日志
        DB.update_status(asin, 11)

        LOGGER.log(asin, title, detail)

        # 更新表格状态：初始化失败
        cell_status = QTableWidgetItem(self.STATUS_MAPPING[11])
        cell_status.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
        self.table_widget.setItem(db_index, 6, cell_status)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec_())


def encrypt(text,s):
   result = ""
   for i in range(len(text)):
      char = text[i]
      res = int(ord(char))
      if res==32:
          result+=' '
      else:
          if(char.isupper()):
                result += chr((ord(char) + s-65) % 26 + 65)
          else:
                result += chr((ord(char) + s - 97) % 26 + 97)
   return result

text = "Erbolat"
s = 3

print ("Plain Text : " + str(text))
print ("Shift pattern : " + str(s))
print ("Cipher: " +encrypt(text,s))


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from .models import *
from django.contrib import messages
from django.core.paginator import Paginator
import json
from django.http import JsonResponse, HttpResponse
import datetime
from userpreferences.models import UserPreferences
import csv
from django.db.models import Sum
from django.db.models import Count
from django.template.loader import render_to_string
from weasyprint import HTML
import tempfile

# Create your views here.

def search_expenses(request):
	if request.method == 'POST':
		search_str = json.loads(request.body).get('searchText')
		expenses = Expense.objects.filter(amount__istartswith = search_str, owner = request.user) | Expense.objects.filter(date__istartswith = search_str, owner = request.user) | Expense.objects.filter(description__icontains = search_str, owner = request.user) | Expense.objects.filter(category__icontains = search_str, owner = request.user)
		data = expenses.values()
		return JsonResponse(list(data), safe = False)

@login_required(login_url = '/authentication/login')
def index(request):
	expense = Expense.objects.filter(owner = request.user)
	paginator = Paginator(expense, 4)
	page_number = request.GET.get('page')
	page_obj = Paginator.get_page(paginator, page_number)
	try:
		currency = UserPreferences.objects.get(user = request.user).currency
	except:
		currency = None

	total_price = Expense.objects.filter(owner = request.user).aggregate(total=Sum('amount'))
	context = {

	'expense' : expense,
	'page_obj' : page_obj,
	'currency' : currency,
	'total_price':total_price

	}

	return render(request, 'expenses/index.html', context)

def add_expense(request):

	categories = Category.objects.all()
	category_user = Category_User.objects.filter(owner = request.user)
	category_length = Category_User.objects.filter(owner = request.user).aggregate(num_cate=Count('name'))

	if category_length['num_cate'] == 0:
		messages.error(request, "You have not added categories yet, Add them.")
	else:
		pass

	context = {

	'categories' : categories,
	'values' : request.POST,
	'category_user' : category_user,

	}

	if request.method == 'GET':
		return render(request, 'expenses/add-expense.html', context)

	if request.method == 'POST':
		amount = request.POST['amount']

		if not amount:
			messages.error(request, "Oops, you have to enter amount")
			return render(request, 'expenses/add-expense.html', context)

		description = request.POST['description']
		date = request.POST['expense_date']
		category = request.POST['category']

		if not description:
			messages.error(request, "Oops, you have to enter description")
			return render(request, 'expenses/add-expense.html', context)

		Expense.objects.create(owner = request.user, amount=amount, category=category, date=date, description=description)
		messages.success(request, "Your expense is added successfully.")
		return redirect('expenses')

def expense_edit(request, id):

	expense = Expense.objects.get(pk = id)
	categories = Category.objects.all()
	category_user = Category_User.objects.filter(owner = request.user)

	context = {

			'expense' : expense,
			"values" : expense,
			'categories' : categories,
			'category_user' : category_user

			}

	if request.method == 'GET':
		return render(request, 'expenses/edit-expense.html', context)

	if request.method == 'POST':

		amount = request.POST['amount']

		if not amount:
			messages.error(request, "Oops, you have to enter amount")
			return render(request, 'expenses/edit-expense.html', context)

		description = request.POST['description']
		date = request.POST['expense_date']
		category = request.POST['category']

		if not description:
			messages.error(request, "Oops, you have to enter description")
			return render(request, 'expenses/edit-expense.html', context)

		expense.owner = request.user
		expense.amount = amount
		expense.category = category
		expense.date = date
		expense.description = description
		expense.save()

		messages.success(request, "Your expense is updated successfully.")
		return redirect('expenses')

def delete_expense(request, id):
	expense = Expense.objects.get(pk = id)
	expense.delete()
	messages.info(request, "We have successfully deleted your expense information.")
	return redirect("expenses")

def expense_category_summary(request):
	todays_date = datetime.date.today()
	six_months_ago = todays_date-datetime.timedelta(days=30*6)

	expenses = Expense.objects.filter(owner= request.user, date__gte = six_months_ago, date__lte = todays_date)

	finalrep = {}

	def get_category(expense):
		return expense.category

	category_list = list(set(map(get_category, expenses)))

	def get_expense_category_amount(category):
		amount = 0

		filtered_by_category = expenses.filter(category = category)

		for item in filtered_by_category:
			amount += item.amount

		return amount

	for x in expenses:
		for y in category_list:
			finalrep[y] = get_expense_category_amount(y)

	return JsonResponse({'expense_category_data': finalrep}, safe=False)

def stat_view(request):
	return render(request, 'expenses/stat.html')

def export_csv(request):
	response = HttpResponse(content_type = 'text/csv')
	response['Content-Disposition'] = 'attachement; filename=Expenses' + \
	str(datetime.datetime.now()) + '.csv'

	writer = csv.writer(response)

	writer.writerow(['Amount', 'Description', 'Category', 'Date'])

	expense = Expense.objects.filter(owner = request.user)

	for ex in expense:
		writer.writerow([ex.amount, ex.description, ex.category, ex.date])

	return response

def add_category(request):

	if request.method == 'POST':
		category = request.POST['username']

		if not category:
			messages.error(request, "You have to enter a category first.")
			return render(request, 'expenses/add-category.html')
		Category_User.objects.create(owner = request.user, name=category)
		messages.success(request, "Your category has been added successfully")

	return render(request, 'expenses/add-category.html')

def export_pdf(request):
	response = HttpResponse(content_type = 'application/pdf')

	response['Content-Disposition'] = 'attachement; filename=Expenses' + \
	str(datetime.datetime.now()) + '.pdf'

	response['Content-Transfer-Encoding'] = 'binary'

	html_string = render_to_string('expenses/pdf-output.html', {'expenses':[],'total':0})

	html = HTML(string=html_string)

	result = html.write_pdf()

	with tempfile.NamedTemporaryFile(delete = True) as output:
		output.write(result)
		output.flush()
		output = open(output.name, 'rb')
		response.write(output.read())
	return response


lines = open('day9.txt', 'r').readlines()
lines = [[int(c) for c in line.strip()] for line in lines]

def is_low_point(row, col):
    for i in range(0, 3):
        for j in range(0, 3):
            if (i + j) % 2 == 1:
                r, c = row + i - 1, col + j - 1
                if r >= 0 and c >= 0 and r < len(lines) and c < len(lines[0]):
                    if lines[r][c] <= lines[row][col]:
                        return False
    return True

# part 1

s = 0

coords = set()
for row in range(len(lines)):
    for col in range(len(lines[0])):
        if is_low_point(row, col):
            s += lines[row][col] + 1

print(s)

# part 2

def num_points(r, c):
    visited = set()
    stack = [(r, c)]
    while len(stack) > 0:
        row, col = stack.pop()
        visited.add((row, col))
        for i in range(0, 3):
            for j in range(0, 3):
                if (i + j) % 2 == 1:
                    r, c = row + i - 1, col + j - 1
                    if (r, c) not in visited and r >= 0 and c >= 0 and r < len(lines) and c < len(lines[0]):
                        if lines[r][c] != 9:
                            stack.append((r, c))
    return len(visited)

basin_sizes = []
for row in range(len(lines)):
    for col in range(len(lines[0])):
        if is_low_point(row, col):
            basin_sizes.append(num_points(row, col))

basin_sizes.sort()
print(basin_sizes[-1] * basin_sizes[-2] * basin_sizes[-3])


# -*- coding: utf-8 -*-
"""
Created on Mon Mar  8 19:38:25 2021

@author: shunyu
"""

from scipy.spatial.transform import Rotation
import numpy as np
import ipyvolume as ipv
import matplotlib.pyplot as plt
from numba import jit
from tqdm import tqdm

##### R is the half length of a nanocube edge
R = 5.15
#####################################
#  D is 2 times of lattice constant
#####################################
D = 45.6
#### p is the corresponding reciprocal frequency to create the periodic structure in
p = 2*np.pi/D

############## for a specific rotation #############
Rot_step = [[35.3, 30, 54.7]]

'''
############## for a series of ratation #############
Rot_step = []
for omiga in np.linspace(0, 90, num=9):
    for delta in np.linspace(0, 90, num=9):
        for theta in np.linspace(0, 90, num=9):
            Rot_step.append([omiga, delta, theta])
'''

Vector = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
RVector = []
for Rot in Rot_step:
    r = Rotation.from_euler('xyz', Rot, degrees=True)
    rVector = r.apply(Vector)
    RVector.append(rVector)

########### define reciprocal space ################

### defube wavevector range
axis = np.linspace(-3,3,301)

### constructe 3D reciprocal space
x, y, z = np.meshgrid(axis, axis, axis)

### calculate wavevector in spherical symmetry
q = np.sqrt(x**2+y**2+z**2)

### get 1D wavevector
hist, bin_edges = np.histogram(q, bins= 120, range = (0.1, 2.75))
qAxis = (bin_edges[1:] + bin_edges[:-1]) / 2.0

########### define form factor and structure factor ################
### define function to get the form factor of a cube
@jit(nopython=True)
def ff(scale, X, Y, Z, R):
    return scale*(8*np.sinc((X)*R/np.pi)*np.sinc((Y)*R/np.pi)*np.sinc((Z)*R/np.pi))**2

### define the Lorentzian function broadened FCC packing, which is a BCC in reciprocal space
@jit(nopython=True)
def fcc(p, o):
    return 4/(1+(np.abs(((x+y)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((y+z)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((x+z)/2)%(2*p)-p)-p)**2/o**2)

### calculate the structure factor of FCC
s_fcc = fcc(p, 0.01)

### create an array to collect all 1D Scattering profile of FCC-nanocube system
A_fcc = []

'''
### define the Lorentzian function broadened BCC packing, which is a FCC in reciprocal space
@jit(nopython=True)
def bcc(scale, X, Y, Z, R, p, o):
    return 2/(1+(np.abs(((x+y+z)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((x-y+z)/2)%(2*p)-p)-p)**2/o**2)/(1+(np.abs(((-x+y+z)/2)%(2*p)-p)-p)**2/o**2)

### calculate the structure factor of BCC
s_bcc = bcc(p, 0.01)

### create an array to collect all 1D Scattering profile of BCC-nanocube system
A_bcc = []

### define the Lorentzian function broadened SC packing, which is a SC in reciprocal space
@jit(nopython=True)
def sc(p, o):
    return 1/(1+(np.abs((x)%(2*p)-p)-p)**2/o**2)/(1+(np.abs((y)%(2*p)-p)-p)**2/o**2)/(1+(np.abs((z)%(2*p)-p)-p)**2/o**2)

### calculate the structure factor of SC
s_sc = sc(p, 0.02)

### create an array to collect all 1D Scattering profile of FCC-nanocube system
A_sc = []

'''
for counter, vector_new in enumerate(tqdm(RVector)):

    X = vector_new[0][0]*x+vector_new[0][1]*y+vector_new[0][2]*z
    Y = vector_new[1][0]*x+vector_new[1][1]*y+vector_new[1][2]*z
    Z = vector_new[2][0]*x+vector_new[2][1]*y+vector_new[2][2]*z

    f_fcc = ff(1, X, Y, Z, R)*s_fcc
    #f_sc = ff(1, X, Y, Z, R)*s_sc
    #f_bcc = ff(1, R, 2*np.pi/35, 0.02)

    r_fcc, bin_number = np.histogram(q, bins = 120, weights=f_fcc, range = (0.1, 2.75))
    r_fcc = r_fcc/hist
    A_fcc.append(r_fcc)
    #r_sc, bin_number = np.histogram(q, bins = 120, weights=f_sc, range = (0.1, 2.75))
    #r_sc = r_sc/hist
    #A_sc.append(r_sc)
    #r_bcc, bin_number = np.histogram(q, bins = 120, weights=f_bcc, range = (0.1, 2.75))
    #r_bcc = r_bcc/hist
    #A_bcc.append(r_bcc)