import base64
import io
import json
import os
import time
from StringIO import StringIO

import PIL
from PIL import Image
from bson import ObjectId
from cognitive_face import CognitiveFaceException
from flask import Flask, make_response, send_file
from flask import render_template, jsonify, request, flash, redirect, Response, url_for
from flask_mongo_sessions import MongoDBSessionInterface
from flask_mongoengine.wtf import model_form
from mongoengine import DoesNotExist
from pymongo.database import Database
from werkzeug.datastructures import FileStorage
from flask_mongoengine import MongoEngine

from model.person import Person
from model.notification import Notification
from model.face import Face, FaceDocument
from model.personlegacy import PersonLegacy
from person_group import PersonGroup
from utils import get_db, JSONEncoder, initialize_cf, IGNORE_PERSON_GROUP, UNKNOWN_PERSON_GROUP, KNOWN_PERSON_GROUP, \
    setup_mongoengine, get_secret
from forms import FaceUploadForm
from notifier_rest import NotifierREST

APP = Flask(__name__)

# This is for XHR
APP.config['SECRET_KEY'] = os.environ['AZURE_CLIENT_SECRET']
# This is bad :(
APP.config['WTF_CSRF_ENABLED'] = False

setup_mongoengine(APP)
MongoEngine(APP)

db = get_db()  # type: Database
APP.session_interface = MongoDBSessionInterface(APP, db, 'sessions')  # Setup session handler using mongo

# TODO: Deprecated
test_collection = db.get_collection('test')
# TODO: Deprecated
new_faces = db.get_collection('new_faces')

face_collection = db.get_collection('faces')  # Images dump
persons_collection = db.get_collection('persons')  # Only known people

# Initialize Cognitive face
try:
    initialize_cf()
except Exception as ex:
    flash('Failed to initialize CF: {}'.format(ex), category='danger')

# use the modified encoder class to handle ObjectId & datetime object while jsonifying the response.
APP.json_encoder = JSONEncoder

@APP.errorhandler(Exception)
def exception_handler(ex):
    return jsonify({'error': ex.message}), 500

@APP.route('/')
def main_page():
    return redirect(url_for('show_all_persons'))

def show_all():
    pending_users = []
    for new_face in new_faces.find():
        object_id = ObjectId(new_face['_id'])
        img_url = url_for('get_face_image', object_id=object_id)

        train_known_face = url_for('train_face_known', object_id=object_id)
        train_ignore_face = url_for('train_face_ignore', object_id=object_id)
        delete_face_url = url_for('delete_face', object_id=object_id)
        person_info_url = url_for('person_info', object_id=object_id)

        status = 'untrained' if new_face['status'] == 'new' else new_face['status']

        pending_users.append(
            {'ts': object_id.generation_time, 'id': object_id, 'img_url': img_url, 'train_known': train_known_face,
             'train_ignore': train_ignore_face, 'delete_face_url': delete_face_url, 'info_url': person_info_url,
             'status': status})

    return render_template('pending_users.html', pending_users=pending_users)

@APP.route('/person/json/<object_id>')
def person_info(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return jsonify(person.document)

@APP.route('/person/identify/<object_id>')
def person_identify(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    possible_matches = []
    for pg in [PersonGroup.known_person_group(), PersonGroup.unknown_person_group(), PersonGroup.ignore_person_group()]:
        identification_result = pg.identify(person)
        if not identification_result:
            continue

        for candidates in identification_result.values():
            for candidate in candidates:
                person_id = candidate['personId']
                matching_person = PersonLegacy.fetch_by_person_id(new_faces, person_id)
                if not matching_person:
                    continue

                possible_matches.append(matching_person.id)

        time.sleep(1)

    if person.id not in possible_matches:
        possible_matches.append(person.id)
    return jsonify(result=possible_matches), 200

@APP.route('/reset_training')
def reset_training():
    PersonGroup.reset_all()
    for face_document in new_faces.find():
        person = PersonLegacy(new_faces, face_document)
        person.remove_all_trained_details()

    flash('Reset all training group', category='info')
    return redirect('/')

@APP.route('/face/known/<object_id>')
def is_face_known(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)

    person_group_id = person.get_group_person_id(KNOWN_PERSON_GROUP)
    result = person_group_id is not None

    if not result:
        result = person in PersonGroup.known_person_group()

    return jsonify({'result': result})

@APP.route('/face/unknown/<object_id>')
def is_face_unknown(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return jsonify({'result': person.get_group_person_id(UNKNOWN_PERSON_GROUP)})

@APP.route('/face/ignored/<object_id>')
def is_face_ignored(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return jsonify({'result': person.get_group_person_id(IGNORE_PERSON_GROUP)})

@APP.route('/face/image/<object_id>')
def get_face_image(object_id):
    person = PersonLegacy.fetch(new_faces, object_id)
    return Response(person.image, mimetype='image/jpeg')

@APP.route('/person/delete/<object_id>')
def delete_face(object_id):  # TODO: Sanitize this input
    person = PersonLegacy.fetch(new_faces, object_id)
    result = person.delete()
    if result is None:
        flash('Failed deleting: {}'.format(json.dumps(result)))

    return redirect('/')

@APP.route('/train_face/ignore/<object_id>')
def train_face_ignore(object_id):
    person_group = PersonGroup.ignore_person_group()
    return train_face(object_id, person_group)

@APP.route('/train_face/known/<object_id>')
def train_face_known(object_id):
    person_group = PersonGroup.known_person_group()
    return train_face(object_id, person_group)

@APP.route('/train_face/unknown/<object_id>')
def train_face_unknown(object_id):
    person_group = PersonGroup.unknown_person_group()
    return train_face(object_id, person_group)

def train_face(object_id, person_group):
    person = PersonLegacy.fetch(new_faces, object_id)
    try:
        person_group.add_person(person)
    except CognitiveFaceException as ex:
        if ex.status_code in [400, 404]:
            flash("Face not detected, can't add to group - " + person_group.name, category='warning')
            return redirect('/')
    except Exception as ex:
        flash("API Failed - {}".format(ex.message), category="danger")
        return redirect("/")

    unknown_person_group = PersonGroup.unknown_person_group()
    if not unknown_person_group.remove_person(person):
        flash("Face doesn't exist in " + unknown_person_group.name + " so it wasn't removed", category='info')
    return redirect("/")

@APP.route('/test')
def test_endpoint():
    try:
        output = []
        for s in new_faces.find():
            output.append(ObjectId(s['_id']))

        return jsonify({'result': output}), 200
    except Exception as ex:
        return ex.message, 500

def _store_uploaded_face():
    if 'faceImage' not in request.files:
        flash('No file uploaded')
        return render_template('add_face.html')

    file = request.files['faceImage']  # type: FileStorage
    if file.filename == '':
        flash('No file selected', category='danger')
        return render_template('add_face.html')

    with file.stream as f:
        b64_file = base64.b64encode(f.read())

    if b64_file is None:
        flash('Failed converting file to b64')
        return render_template('add_face.html')

    try:
        result = new_faces.insert({'image': b64_file, 'status': 'new'})
    except Exception as ex:
        flash(ex.message)
        return render_template('add_face.html')

    return redirect('/')

@APP.route('/add_face', methods=['GET', 'POST'])
def add_face():
    if request.method == 'POST':
        return _store_uploaded_face()
    else:
        return render_template('add_face.html')

def set_person_id(template, id):
    response = make_response(template)
    if id:
        response.headers['X-Person-ID'] = id

    return response

@APP.route('/face/create', methods=['POST', 'GET'])
def face_store():
    id = request.args.get('id', None)
    id_param = '?id=' + id if id else ''

    fuf = FaceUploadForm()

    response_back_to_face_create = make_response(
        render_template('face_add_form.html', form=fuf, person_id_param=id_param))
    if id:
        response_back_to_face_create.headers['X-Person-ID'] = id

    if fuf.is_submitted():
        return handle_face_upload(fuf, id, response_back_to_face_create)

    return response_back_to_face_create

def handle_face_upload(fuf, id, response_back_to_face_create):
    face = FaceDocument(image=fuf.file.data.read())
    flash('Added face id ' + str(id), category='info')

    try:
        person = Person.objects.get(id=id) if id else None
    except DoesNotExist as ex:
        person = None

    if not person:
        flash("Can't create face without person context '{}'".format(id), category='warning')
        return redirect(url_for('create_person'))

    face.person = person.id
    known_group = PersonGroup.known_person_group()

    try:
        result = known_group.add_face_to_person(person.person_id, face)
    except CognitiveFaceException as ex:
        flash(ex.msg, category='danger')
        return response_back_to_face_create

    face.save()
    person.trained_faces = person.trained_faces + [str(face.id)]  # Lists are immutable
    person.save()
    known_group.train()  # This shouldn't happen all the time but whatever
    return redirect(url_for('get_person', object_id=str(person.id)))

# @APP.route('/face_migrate')
# def face_document_migrate():
#     iter = face_collection.find(projection={'image': False})
#     for document in iter:
#         old_face = Face.find(face_collection, document['_id'])
#         new_face = FaceDocument(id=old_face.id, image=old_face.read(), person=old_face.person)
#         new_face.save()
#
#     return 'Ok', 200

@APP.route('/face/<object_id>')
def face_get(object_id):
    face = FaceDocument.objects.get(id=object_id)

    width = request.args.get('w', default=None)
    width = int(width) if width else None
    height = request.args.get('h', default=None)
    height = int(height) if height else None

    if not width and not height:
        return Response(face.image.read(), mimetype='image/jpeg')

    img = Image.open(face.image)

    if width:
        wpercent = (width / float(img.size[0]))
        height = int((float(img.size[1]) * float(wpercent)))
    elif height:
        hpercent = (height / float(img.size[1]))
        width = int((float(img.size[0]) * float(hpercent)))
    else:
        return "Error", 500

    roiImg = img.resize((width, height), PIL.Image.ANTIALIAS)  # we should probably cache this
    img_io = StringIO()
    roiImg.save(img_io, 'JPEG', quality=80)
    img_io.seek(0)
    return send_file(img_io, mimetype='image/jpeg')

@APP.route('/face/delete/<object_id>')
def face_delete(object_id):
    # Face.delete(face_collection, ObjectId(object_id))
    flash('Deleted face ' + str(object_id) + " didn't really happen")
    return redirect(url_for('show_all_faces'))

@APP.route('/face/update/<object_id>')
def face_update(object_id):
    pass  # http://wtforms.simplecodes.com/docs/0.6.1/fields.html (look at "Select fields with dynamic choice value")

@APP.route('/face')
def show_all_faces():
    faces = Face.find_all(face_collection)
    return render_template('show_all_faces.html', faces=faces)

@APP.route('/person/')
def show_all_persons():
    all_persons = Person.objects()
    return render_template('show_all_persons.html', persons=all_persons)

@APP.route('/person/<object_id>')
def get_person(object_id):
    person = Person.objects.get(id=object_id)
    return render_template('show_person.html', person=person)

@APP.route('/person/create', methods=['GET', 'POST'])
def create_person():
    PersonForm = model_form(Person)

    form = PersonForm(request.form)
    if request.method == 'POST' and form.validate():
        # Create person for PersonGroup
        person_name = form.name.data
        try:
            person = Person.objects.get(name=person_name)
        except DoesNotExist as ex:
            person = None

        if person:
            flash('Person {} already exists, not creating a new one'.format(person_name), category='info')
        else:
            person_id = PersonGroup.known_person_group().add_person_by_name(person_name)
            form.person_id.data = person_id
            form.person_group = PersonGroup.known_person_group().name
            person = form.save(validate=False)  # type: Person
            flash('Person {} successfully save'.format(person_name), category='info')

        icum_person_id = str(person.id)
        response = redirect(url_for('face_store') + '?id=' + icum_person_id)
        response.headers['X-Person-ID'] = icum_person_id

        return response

    return render_template('person_form.html', form=form)

@APP.route('/person/update/<object_id>')
def update_person(object_id):
    pass

@APP.route('/person/delete/<object_id>')
def delete_person(object_id):
    pass

@APP.route('/notifications/<notification_id>')
def show_notification(notification_id):  # Show in mobile page
    notification = Notification.objects.get(id=notification_id)
    time = notification.id.generation_time
    time_str = "{}-{}-{}, {}:{}".format(time.day, time.month, time.year, str(time.hour).zfill(2), str(time.minute).zfill(2))
    face_id = notification.icum_face_id
    face = FaceDocument.objects.get(id=face_id)
    # person = Person.objects.get(id=face.person) if face.person else None
    return render_template('show_notification.html', notification=notification, face=face, person=face.person, time=time_str)
    # return "we recognized someone at your door!\n details:{}".format(pretty)

@APP.route('/notifications/<notification_id>/dismiss', methods=['POST'])
def dismiss(notification_id):
    pass

@APP.route('/notifications/<notification_id>/add', methods=['POST'])
def new_person_from_notification(notification_id):  # Create a new person from notification image
    pass

# Dummy method to create notification online
@APP.route('/notifications/create', methods=['GET', 'POST'])
def create_notification():  # Create a new person from notification image
    NotificationForm = model_form(Notification, field_args={'msg': {'textarea': True}})

    form = NotificationForm(request.form)
    if request.method == 'POST' and form.validate():
        form.save(validate=False)
        flash('success!', category='success')

    return render_template('add_notification.html', form=form)

@APP.route('/notifications/')
def show_all_notifications():
    all_notifications = Notification.objects()
    return render_template('show_all_notifications.html', notifications=all_notifications)

def _handle_face_notification(icum_face_id, person=None, attributes=None):
    """
    No person implies that the person is unknown
    :type icum_face_id: str
    :type person: Person
    """
    if not person:
        msg = "Unknown person detected !"
        msg_type = 'unknown'
        attributes = attributes
    else:
        msg = "Person '{}' detected !".format(person.name)
        msg_type = 'known'

    notification = Notification(msg=msg, icum_face_id=icum_face_id, msg_type=msg_type, attributes=attributes)
    notification.save()

    event_hub_connection_string = get_secret("NOTIFICATION-HUB-CONNECTION-STRING")
    notifier = NotifierREST(event_hub_connection_string)
    return notifier.notify(notification)

@APP.route('/detect', methods=['POST'])
def detect():
    face_bytes = request.get_data()
    face = FaceDocument(image=face_bytes)
    face.save()

    knowns = PersonGroup.known_person_group()
    detected_face = knowns.detected_face(face.image)  # TODO: Fix for multiple people
    detected_face_guid = detected_face['id']

    identified_dict = knowns.identify_face(detected_face_guid)

    candidates = identified_dict[detected_face_guid]

    most_suitable = max(candidates, key=lambda record: record['confidence']) if candidates else None
    person = Person.objects.get(person_id=most_suitable['personId']) if candidates else None
    face.person = person.id if person else None
    face.save()
    icum_face_id = str(face.id)

    if not person:
        _handle_face_notification(icum_face_id, attributes=detected_face['attributes'])
        return jsonify(status="person is unknown"), 404

    msg = _handle_face_notification(icum_face_id, person)
    return jsonify(candidates=candidates, status='Notified person icum id {}'.format(str(person.id))), msg.status

if __name__ == '__main__':
    APP.run(debug=True)


from modules import connection
from modules import formatter
from tabulate import tabulate

def authors():

    db_connection = connection.get_connection()
    cursor = db_connection.cursor()
    popular_authors = """
        SELECT name,title,path,count(title) AS views
        FROM articles,log,authors
        WHERE log.path = CONCAT('/article/', articles.slug)
        AND articles.author = authors.id
        GROUP BY path, title, name ORDER BY views DESC LIMIT 3;"""

    cursor.execute(popular_authors)
    return cursor.fetchall()

def print_authors():
    print("Top Authors:")
    print()
    print(tabulate([
        {
            "Title": title,
            "Author": author,
            "Page": page,
            "Numbers of views": result
        } for (author, title, page, result) in authors()], headers="keys")) # MODIFY EVERY ELEMENT
    print()


import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams

import numpy as np

import constants
import input_parts
from relshift_lstm import RelativeShiftLSTMStack
from queue_managers import QueueManager
from adam import Adam
from note_encodings import Encoding
import leadsheet

import itertools
import functools
from theano_lstm import LSTM, StackedCells, Layer
from util import *
import random

import pickle

CHARKEY = " !\"'(),-.01245679:?ABCDEFGHIJKLMNOPQRSTUVWYZabcdefghijklmnopqrstuvwxyz"

def name_model():

    LSTM_SIZE = 300
    layer1 = LSTM(len(CHARKEY), LSTM_SIZE, activation=T.tanh)
    layer2 = Layer(LSTM_SIZE, len(CHARKEY), activation=lambda x:x)
    params = layer1.params + [layer1.initial_hidden_state] + layer2.params

    ################# Train #################
    train_data = T.ftensor3()
    n_batch = train_data.shape[0]
    train_input = T.concatenate([T.zeros([n_batch,1,len(CHARKEY)]),train_data[:,:-1,:]],1)
    train_output = train_data

    def _scan_train(last_out, last_state):
        new_state = layer1.activate(last_out, last_state)
        layer_out = layer1.postprocess_activation(new_state)
        layer2_out = layer2.activate(layer_out)
        new_out = T.nnet.softmax(layer2_out)
        return new_out, new_state

    outputs_info = [None, initial_state(layer1, n_batch)]
    (scan_outputs, scan_states), _ = theano.scan(_scan_train, sequences=[train_input.dimshuffle([1,0,2])], outputs_info=outputs_info)

    flat_scan_outputs = scan_outputs.dimshuffle([1,0,2]).reshape([-1,len(CHARKEY)])
    flat_train_output = train_output.reshape([-1,len(CHARKEY)])
    crossentropy = T.nnet.categorical_crossentropy(flat_scan_outputs, flat_train_output)
    loss = T.sum(crossentropy)/T.cast(n_batch,'float32')

    adam_updates = Adam(loss, params)

    train_fn = theano.function([train_data],loss,updates=adam_updates)

    ################# Eval #################

    length = T.iscalar()
    srng = MRG_RandomStreams(np.random.randint(1, 1024))

    def _scan_gen(last_out, last_state):
        new_state = layer1.activate(last_out, last_state)
        layer_out = layer1.postprocess_activation(new_state)
        layer2_out = layer2.activate(layer_out)
        new_out = T.nnet.softmax(T.shape_padleft(layer2_out))
        sample = srng.multinomial(n=1,pvals=new_out)[0,:]
        sample = T.cast(sample,'float32')
        return sample, new_state

    initial_input = np.zeros([len(CHARKEY)], np.float32)
    outputs_info = [initial_input, layer1.initial_hidden_state]
    (scan_outputs, scan_states), updates = theano.scan(_scan_gen, n_steps=length, outputs_info=outputs_info)

    gen_fn = theano.function([length],scan_outputs,updates=updates)

    return layer1, layer2, train_fn, gen_fn

def train_name(dataset_file):
    with open(dataset_file,'r') as f:
        dataset = [x.strip() for x in f]
    maxlen = max(len(x) for x in dataset)
    dataset = [x+" "*(maxlen-len(x)) for x in dataset]

    layer1, layer2, train_fn, gen_fn = name_model()
    params = layer1.params + [layer1.initial_hidden_state] + layer2.params

    print("Starting train...")

    BATCH_SIZE = 20
    for iteration in range(10000):
        sample = [random.choice(dataset) for _ in range(BATCH_SIZE)]
        sample_encoded = np.zeros([BATCH_SIZE, maxlen, len(CHARKEY)], np.float32)
        for i,train_string in enumerate(sample):
            for j,c in enumerate(train_string):
                try:
                    idx = CHARKEY.index(c)
                except ValueError:
                    print("Couldn't find character <{}>, replacing with space".format(c))
                    idx = len(CHARKEY)-1
                sample_encoded[i,j,idx] = 1.0

        loss = train_fn(sample_encoded)
        if iteration % 100 == 0:
            print("Iter",iteration,"has loss",loss)
            for _ in range(10):
                generate_name(maxlen, layer1, layer2, train_fn, gen_fn)

    pickle.dump([p.get_value() for p in params], open("name_params.p", 'wb'))

def generate_name(length, layer1=None, layer2=None, train_fn=None, gen_fn=None, num_names=1):
    if layer1 is None:
        layer1, layer2, train_fn, gen_fn = name_model()
        params = layer1.params + [layer1.initial_hidden_state] + layer2.params
        loaded_params = pickle.load(open("name_params.p", 'rb'))
        for p,v in zip(params, loaded_params):
            p.set_value(v)
    for i in range(num_names):
        scan_outputs = gen_fn(length)
        outval = []
        for output in scan_outputs:
            idx = np.nonzero(output)[0][0]
            outval.append(CHARKEY[idx])
        print(''.join(outval))


"""Intialise the counter i=j=first element of array and pivot=last element of array
   if array[j]< pivot swap array(i,j) and increment i
   last swap array(i,pivot) element
   this algorithm give in place 0(n) partitioning in 0(1) extra memory
"""

def pivot(array, a, b):
    i = a
    x = array[b - 1]
    for j in range(a, b - 1):
        if array[j] < x:
            array[i], array[j] = array[j], array[i]
            i += 1
    array[i], array[b - 1] = array[b - 1], array[i]
    return i

'''This Quicksort algorithms work on randomise input
   we can do better introducing randomisation to algorithm
'''

def Quicksort(array, start, end):
    if start < end:
        q = pivot(array, start, end)
        Quicksort(array, start, q)
        Quicksort(array, q + 1, end)

array=[1,3,4,6,9,7,5]
print(pivot(array,0,6))
print(array)


import logging

import rbac_auth
import rbac_exceptions
from tempest import config

from tempest.lib import exceptions

CONF = config.CONF
LOG = logging.getLogger(__name__)

def action(component, service, rule):
    def decorator(func):
        def wrapper(*args, **kwargs):
            authority = rbac_auth.RbacAuthority(
                CONF.rbac.rbac_policy_file, component, service)
            allowed = authority.get_permission(rule, CONF.rbac.rbac_role)
            try:
                func(*args)
            except exceptions.Forbidden as e:
                if allowed:
                    msg = ("Role %s was not allowed to perform %s." %
                           (CONF.rbac.rbac_role, rule))
                    LOG.error(msg)
                    raise exceptions.Forbidden(
                        "%s exception was: %s" %
                        (msg, e))
            except rbac_exceptions.RbacActionFailed as e:
                if allowed:
                    msg = ("Role %s was not allowed to perform %s." %
                           (CONF.rbac.rbac_role, rule))
                    LOG.error(msg)
                    raise exceptions.Forbidden(
                        "%s RbacActionFailed was: %s" %
                        (msg, e))
            else:
                if not allowed:
                    LOG.error("Role %s was allowed to perform %s" %
                              (CONF.rbac.rbac_role, rule))
                    raise StandardError(
                        "OverPermission: Role %s was allowed to perform %s" %
                        (CONF.rbac.rbac_role, rule))
        return wrapper
    return decorator


import multiprocessing
import multiprocessing.pool
import traceback
import time
import gc

from . names import *
from . derived import *
from . cloud import *
from . common import timed

def addFields(df, name_dict):
    for new, orig in name_dict.items():
        df[new] = df[orig]

def normalizeFields(p, show_fields=False, allow_missing=False):
    names = p.columns.tolist()
    # Ensure the alphanumerical model IDs are interpreted as strings
    p.Model = p.Model.astype('str')

    toner = findFields(names, '.*Toner.Bottle.%s.Remaining.Toner.(?!previous).*','Toner.%s', allow_missing=allow_missing)
    addFields(p, toner)

    try:
        toner_bw =  findFields(names, '.*Toner.status.Percentage','Toner.K', allow_missing=allow_missing,colors=False)

        addFields(p,toner_bw)
        toner.update(toner_bw)
    except:
        pass

    toner_names = list(toner.keys())

    toner_end = findFields(names, '.*Toner.Bottle.%s.End.SP7.*', 'Toner.End.Status.%s', allow_missing=allow_missing)
    addFields(p, toner_end)

    toner_end_date = findFields(names, '.*Toner.Bottle.%s.End.Date.*', 'Toner.End.Date.%s', allow_missing=allow_missing)
    addFields(p, toner_end_date)

    pages = findFields(names, '.*Pages.Current.Toner.%s.(?!previous).*', 'Pages.Toner.%s', allow_missing=allow_missing)
    addFields(p, pages)

    prev_pages = findFields(names, '.*Pages.Current.Toner.%s.previous.*', 'Pages.Previous.Toner.%s', allow_missing=allow_missing)
    addFields(p, prev_pages)

    cov = findFields(names, '.*Pixel.Coverage.Accumulation.Coverage.%s.*', 'Coverage.%s', allow_missing=allow_missing)
    addFields(p, cov)
    cov_names = list(cov.keys())

    dev_replacement = findFields(
        names,
        ['.*Replacement.Date.Developer.%s.*',
         '.*Unit.Replacement.Date.Dev.Unit.%s.*'],
         'Developer.Replacement.Date.Recorded.%s',
         allow_missing=True,
    )
    addFields(p, dev_replacement)
    # Convert 'yymmdd' representation of dev unit replacement dates to datetime
    for f in dev_replacement.keys():
        p[f] = pd.to_datetime(p[f], format='%y%m%d', errors='coerce')

    pcus = findFields(names, '.*Estimated.Usage.Rate.PCU.%s', 'PCU.Yield.%s', allow_missing=allow_missing)
    addFields(p, pcus)
    pcus_names = list(pcus.keys())

    total_pages = findFields(names, '.*Total.Total.PrtPGS.SP8.*', 'Pages', colors=False, allow_missing=allow_missing)
    addFields(p, total_pages)
    pages_names = list(total_pages.keys())

    copy_bw_pages = findFields(names, '.*Total.PrtPGS.Color.Mode.B.W.SP8.*', 'Pages.Copy.BW', colors=False, allow_missing=allow_missing)
    addFields(p, copy_bw_pages)
    pages_names += list(copy_bw_pages.keys())

    print_bw_pages = findFields(names, '.*Total.PrtPGS.Print.Color.Mode.B.W.SP8.*', 'Pages.Print.BW', colors=False, allow_missing=allow_missing)
    addFields(p, print_bw_pages)
    pages_names += list(print_bw_pages.keys())

    # Include jobs with pages as same requirements and semantically similar
    total_jobs = findFields(names, '.*Total.Total.Jobs.SP8.*', 'Jobs', colors=False, allow_missing=allow_missing)
    addFields(p, total_jobs)
    pages_names = list(total_jobs.keys())

    used_bottles = findFields(names, ['.*used.cartridge.Total.%s',
                                      '.*Toner.Use.Count.%s'],'Toner.Bottles.Total.%s', allow_missing=allow_missing)
    addFields(p, used_bottles)
    used_bottles_names = list(used_bottles.keys())

    dev_rotation = findFields(
        names,
        ['.*(?<!Previous.Unit).PM.Counter.Rotation.Developer.%s.*',
         '.*Display.Distance.Dev.Unit.%s.SP7.942',
         '.*Drive.Distance.Counter.%s_Developer.*',
        ],
        'Developer.Rotation.%s',
        take_first=True,
        allow_missing=allow_missing,
    )
    addFields(p, dev_rotation)

    try:
        bw_dev_rotation= findFields(names, '.*Drive.Distance.Counter.Development.*','Developer.Rotation.K',allow_missing=allow_missing,colors=False)

        addFields(p,bw_dev_rotation)
        dev_rotation.update(bw_dev_rotation)
    except:
        pass

    dev_rotation_names = list(dev_rotation.keys())

    # Always allow missing for toner call threshold as this comes in mutually exclusive variants
    toner_call_threshold = findFields(names, '.*Toner.Call.Threshold.SP5.*', 'Toner.Call.Threshold', colors=False, allow_missing=True)
    addFields(p, toner_call_threshold)
    toner_call_threshold_colored = findFields(names, '.*Toner.Call.Threshold.%s.*', 'Toner.Call.Threshold.%s', colors=['K', 'CMY'], allow_missing=True)
    addFields(p, toner_call_threshold_colored)
    toner_call_timing = findFields(names, '.*Toner.Call.Timing.*', 'Toner.Call.Timing', colors=False, allow_missing=allow_missing)
    addFields(p, toner_call_timing)

    if show_fields:
        print("Toner level fields:")
        print(toner)
        print("Toner end status:")
        print(toner_end)
        print("Toner end date:")
        print(toner_end_date)
        print("Current toner pages:")
        print(pages)
        print("Previous toner pages:")
        print(prev_pages)
        print("Coverage:")
        print(cov)
        print("Dev unit replacement:")
        print(dev_replacement)
        print("Dev unit rotations:")
        print(dev_rotation)
        print("Total Pages:")
        print(total_pages)
        print("Used Bottles:")
        print(used_bottles)
        print("Dev unit rotations:")
        print(dev_rotation)
        print("PCUs:")
        print(pcus)

    return toner_names, cov_names, pages_names, used_bottles_names, dev_rotation_names, pcus_names

# Use global variables for shared data as workaround for limitations of multiprocessing module
# Note that this means that processFile() is not thread-safe.
process_df = None
def applyColorSetInner(args):
    f, color = args
    return f(process_df, color)

def applyColorSet(f, colors):
    with multiprocessing.Pool(len(colors)) as pool:
        res_parts = pool.map(applyColorSetInner, [(f, color) for color in colors], chunksize=1)
    res = pd.DataFrame()
    for part in res_parts:
        #print(part)
        duplicated = set(res.columns).intersection(part.columns)
        if duplicated:
            raise Exception(f"Duplicated columns: {', '.join(duplicated)}")
        res = pd.concat([res, part], axis=1, sort=False)
    return res

def processFile(s3_url, show_fields=False, keep_orig=False, allow_missing=False, toner_stats=True, nz_only=False, in_bucket=None, skip_processing=False, require_color_match=True):
    global process_df

    start_time = time.time()

    # This is memory intensive, so garbage collect to free up any lingering allocations
    gc.collect()

    def status(text):
        now = time.time()
        delta_s = now - start_time
        print(f"{text} [{delta_s:.2f}s elapsed for {s3_url}]")

    status("Reading data from S3")
    p = readFromS3(s3_url)

    # At time of writing the RNZ Falcon files contain data for both AU and NZ machines. This is a workaround to restrict the result to NZ machines where required.
    # TODO: implement by checking if serial is in the regional MRP file rather than using exported NZ DB content (this should be general as well as more current)
    if nz_only:
        p = p[p.Serial.isin(process_nz_sers)]

    if skip_processing:
        return p

    orig_fields = p.columns

    status("Normalizing field names")
    toner_names, cov_names, pages_names, used_bottles_names, dev_rotation_names, pcus_names = normalizeFields(p, show_fields, allow_missing)
    colors_matched = [c for c in colors_norm if f'Toner.{c}' in toner_names]
    print(colors_matched)
    if require_color_match and not colors_matched:
        raise Exception(f"No colors matched for {s3_url}")

    if not keep_orig:
        print("Dropping original fields")
        to_use = set(p.columns).difference(orig_fields)
        # Keep these regardless
        to_use.update([
            'Serial',
            'Model',
            'RetrievedDate',
            'RetrievedDateTime',
            'FileDate',
        ])
        p = p[to_use]

    # Derive total BW and color page counts
    print("Adding page count stats")
    p['Pages.BW'] = p['Pages.Print.BW'] + p['Pages.Copy.BW']
    p['Pages.Color'] = p['Pages'] - p['Pages.BW']
    pages_names += ['Pages.BW', 'Pages.Color']

    # Add delta and replacements
    times = ['RetrievedDate', 'RetrievedDateTime']
    to_add = list(toner_names + cov_names + times + pages_names + used_bottles_names + dev_rotation_names + pcus_names)
    time_deltas = [x+'.delta' for x in times]

    status(f"Adding deltas, rates and replacement fields for {s3_url}")
    addDeltas(p, to_add)
    # Coerce values to timedeltas / NaT
    for d in time_deltas:
        p[d] = pd.to_timedelta(p[d])
    p['RetrievedDateTime.delta.days']=p['RetrievedDateTime.delta'] / datetime.timedelta(days=1)
    addRates(p, to_add)
    addReplacements(p, toner_names)
    # Toner counts down passage of time, PCUs count up - so a negative delta indicates a replacement
    addReplacements(p, pcus_names, negative=True)

    process_df = sortReadings(p)

    if colors_matched and toner_stats:
        def apply(f):
            global process_df
            res = applyColorSet(f, colors_matched)
            process_df = pd.concat([process_df, res], axis=1, sort=False)

        status("Adding toner bottle indices")
        apply(indexToner)
        if pcus_names:
            status("Adding PCU indices")
            apply(indexPCU)

        status("Adding final page count for current toner")
        apply(getTonerPages)

        status("Deriving dev unit replacement dates")
        apply(deriveDevReplacements)
        apply(selectDevReplacementDate)

        status("Project pages for toner bottles and calculate ratios vs. actual pages")
        apply(projectPages)

        status("Project coverage for toner bottles and calculate ratios vs. estimated coverage")
        apply(projectCoverage)

        status("Add toner usage ratios")
        apply(getTonerRatio)

    res = process_df
    process_df = None
    status("Finished")
    return res

def summarizeTonerStats(p):
    print(f"Generating summary for {s3_url}")
    x=p[['Serial','RetrievedDateTime.delta.days']+replacement_fields].groupby('Serial')
    summary=x.sum()

    addSumRate(summary, replacement_fields)
    replacement_rates = [name+".rate" for name in replacement_fields]
    return p

def sortReadings(df):
    df.sort_values(['Serial', 'RetrievedDateTime'], inplace=True, ascending=False)
    return df

def indexToner(df, color='K'):
    prev = df.shift(-1)
    new_ser = df.Serial != prev.Serial
    new = new_ser | df[f'Toner.{color}.replaced']
    new_labels = df.index
    f = f'TonerIndex.{color}'
    res = new_labels.to_frame(name=f)
    res[f] = new_labels.where(new)
    res[f] = res[f].fillna(method='bfill')
    return res

def indexPCU(df, color='K'):
    prev = df.shift(-1)
    new_ser = df.Serial != prev.Serial
    new = new_ser | df[f'PCU.Yield.{color}.replaced']
    new_labels = df.index
    f = f'PCUIndex.{color}'
    res = new_labels.to_frame(name=f)
    res[f] = new_labels.where(new)
    res[f] = res[f].fillna(method='bfill')
    return res

def deriveDevReplacements(df, color='K'):
    rotation = f'Developer.Rotation.{color}'
    prev = df.shift(-1)
    new = df[rotation] < prev[rotation]
    new[-1] = True
    dates = pd.to_datetime(df.RetrievedDate)
    f = f'Developer.Replacement.Date.Derived.{color}'
    res = dates.to_frame(name=f)
    res[f] = dates.where(new)
    res[f] = res[f].fillna(method='bfill')
    return res

def selectDevReplacementDate(df, color='K'):
    priorities = [
        f'Developer.Replacement.Date.Recorded.{color}',
        f'Developer.Replacement.Date.Derived.{color}',
    ]
    for f in priorities:
        if f in df.columns:
            df[f'Developer.Replacement.Date.{color}'] = df[f]
            break
    return df[[f'Developer.Replacement.Date.{color}']]

# Take only rows with summary stats for toner bottles
def selectTonerStats(df):
    colnames = df.columns[df.columns.str.contains('^Projected.Pages.*')]
    cols = df[colnames].dropna(how='all')
    res = df.loc[cols.index]
    return res

def doProcessing(args, summary_rows_only=False):
    path, kwargs = args
    try:
        bucket_name = kwargs.get('in_bucket', in_bucket_name)
        res = processFile(f"s3://{bucket_name}/{path}", **kwargs)
        if summary_rows_only:
            res = selectTonerStats(res)
    except:
        print(f"Exception processing {path}")
        traceback.print_exc()
        res=None
    return res

# https://stackoverflow.com/questions/6974695/python-process-pool-non-daemonic
class NoDaemonProcess(multiprocessing.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)

class NonDaemonPool(multiprocessing.pool.Pool):
    Process = NoDaemonProcess

# Global to make NZ serial list available for children - TODO: replace with MRP data
process_nz_sers = None

@timed
def buildDataset(to_use, kwargs={}, num_procs=None, f=doProcessing):
    if num_procs is None:
        num_procs = int(np.ceil(multiprocessing.cpu_count() / 4))
    global process_nz_sers
    ser_df = pd.read_csv('s3://ricoh-prediction-misc/mif/current/customer.csv')
    process_nz_sers = ser_df.SerialNo

    args = [(path, kwargs) for path in to_use]
    # Use non-daemonic pool as we want children to be able to fork
    with NonDaemonPool(num_procs) as pool:
        res_parts = pool.map(doProcessing, args, chunksize=1)
    res_parts = [x for x in res_parts if x is not None]
    print("Combining result parts")
    res = pd.concat(res_parts)
    return res


def solve(A, B):
    n=len(A)
    for i in range(2**n):
        total_sum=0
        for j in range(n):
            if (i>>j)&1:
                total_sum+=A[j]
        if total_sum==B:
            return 1
    return 0

A = [1, 20, 13, 4, 5]
B = 18
print(solve(A,B))

# output:
# 1

# Sum of subsequence [1,13,4] is 18


class Solution:
    def maxSlidingWindow(self, nums: List[int], k: int) -> List[int]:
        left = 0
        right = 0
        dequeToUse = deque()
        result = []
        while right < len(nums):
            while dequeToUse and dequeToUse[-1] < nums[right]:
                dequeToUse.pop()
            dequeToUse.append(nums[right])
            right += 1

            if right - left == k:
                result.append(dequeToUse[0])
                if dequeToUse[0]== nums[left]:
                    dequeToUse.popleft()
                left += 1

        return result


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Imports resolvers into a database from the public-dns.info data"""

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import argparse
import csv
import logging
import re
import requests
import sys
import dnsresolver
import resolver_db
log = logging.getLogger(__name__)

IPV4_RE = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
THRESHOLD = 0.99

def resolver_publicdns(args: argparse.Namespace):
    """
    Test a list of IP addresses and add valid addresses to the database file
    """
    # Get a database for the resolvers
    db = resolver_db.ResolverDB(args.database)

    if args.publicdns == "global":
        # Special target - download all servers
        target = "https://public-dns.info/nameservers.csv"
    else:
        # Download the associated target CSV file
        target = ("http://public-dns.info/nameserver/{filename}.csv"
                  .format(filename=args.publicdns))

    log.info("Downloading target: %s", target)

    r = requests.get(target)

    # If the download failed, exit here instead
    r.raise_for_status()

    # The CSV data is in hand. Use the CSV reader to read it.
    decoded_content = r.content.decode('utf-8')

    all_resolvers = list(csv.DictReader(decoded_content.splitlines(), delimiter=','))
    valid_resolvers = [dnsresolver.DNSResolver(res["ip"])
                       for res in all_resolvers
                       if res["reliability"] and
                       float(res["reliability"]) >= THRESHOLD and
                       IPV4_RE.match(res["ip"])]
    db.set_valid_many(valid_resolvers)

    log.info("Imported %d valid resolvers out of %d", len(valid_resolvers), len(all_resolvers))

    return ScriptRC.SUCCESS

def main():
    """Main handling function. Wraps resolver_import."""

    stdout_fmt = logging.Formatter("%(asctime)s %(levelname)-5.5s %(message)s")
    stdout_handler = logging.StreamHandler(stream=sys.stdout)
    stdout_handler.setFormatter(stdout_fmt)
    stdout_handler.setLevel(logging.INFO)

    file_fmt = logging.Formatter("%(asctime)s %(levelname)-5.5s %(message)s")
    file_handler = logging.FileHandler(filename="import.log",
                                       mode="w")
    file_handler.setFormatter(file_fmt)
    file_handler.setLevel(logging.DEBUG)

    root_logger = logging.getLogger()
    root_logger.addHandler(stdout_handler)
    root_logger.addHandler(file_handler)
    root_logger.setLevel(logging.DEBUG)

    # Run main script.
    try:
        parser = argparse.ArgumentParser()
        parser.add_argument("--database", help="Database filename (SQLite)", required=True)
        parser.add_argument("--publicdns", help="Which CSV file to download from http://public-dns.info",
                            required=True)

        args = parser.parse_args()
        rc = resolver_publicdns(args)

    except Exception as e:
        log.exception(e)
        rc = ScriptRC.EXCEPTION

    log.info("Returning %d", rc)
    return rc

class ScriptRC(object):
    SUCCESS = 0
    FAILURE = 1
    EXCEPTION = 2

class ScriptException(Exception):
    pass

if __name__ == '__main__':
    sys.exit(main())


import math
import numpy as np
import coalpy.gpu as gpu

from src import Utility
from src import Budgets
from src import PrefixSum
from src import Rasterizer

# Stage Kernels
s_raster_bin            = gpu.Shader(file="RasterBin.hlsl",     name="RasterBin",     main_function="RasterBin")
s_raster_bin_tes        = gpu.Shader(file="RasterBin.hlsl",     name="RasterBin",     main_function="RasterBin", defines=["RASTER_CURVE"])
s_raster_fine           = gpu.Shader(file="RasterFine.hlsl",    name="RasterFine",    main_function="RasterFine")
s_raster_fine_tes       = gpu.Shader(file="RasterFine.hlsl",    name="RasterFine",    main_function="RasterFine", defines=["RASTER_CURVE"])
s_raster_fine_oit       = gpu.Shader(file="RasterFineOIT.hlsl", name="RasterFineOIT", main_function="RasterFineOIT")
s_raster_fine_oit_tes   = gpu.Shader(file="RasterFineOIT.hlsl", name="RasterFineOIT", main_function="RasterFineOIT", defines=["RASTER_CURVE"])
s_build_work_queue_args = gpu.Shader(file="WorkQueue.hlsl",     name="WorkQueueArgs", main_function="BuildWorkQueueArgs")
s_build_work_queue      = gpu.Shader(file="WorkQueue.hlsl",     name="WorkQueue",     main_function="BuildWorkQueue")

class RasterizerBinned(Rasterizer.Rasterizer):
    def __init__(self, w, h):

        self.bin_w = math.ceil(w / Budgets.TILE_SIZE_BIN)
        self.bin_h = math.ceil(h / Budgets.TILE_SIZE_BIN)

        # Resources
        self.b_bin_records = None
        self.b_bin_records_counter = None
        self.b_bin_counters = None
        self.b_bin_min_z = None
        self.b_bin_max_z = None
        self.b_bin_offsets = None
        self.b_work_queue = None
        self.b_work_queue_args = None
        self.b_prefix_sum_args = None

        # Constant buffers
        self.cb_raster_bin = None
        self.cb_raster_fine = None

        # Will invoke the creation of resources
        super().__init__(w, h)

    def create_resource_buffers(self):
        super().create_resource_buffers()

        self.b_bin_records = gpu.Buffer(
            name="BinRecords",
            type=gpu.BufferType.Structured,
            stride=Budgets.BYTE_SIZE_BIN_RECORD_FORMAT,
            element_count=math.ceil(Budgets.BYTE_SIZE_BIN_RECORD_POOL / Budgets.BYTE_SIZE_BIN_RECORD_FORMAT)
        )

        self.b_bin_records_counter = gpu.Buffer(
            name="BinRecordsCounter",
            type=gpu.BufferType.Standard,
            format=gpu.Format.R32_UINT,
            element_count=1
        )

        self.b_work_queue = gpu.Buffer(
            name="WorkQueue",
            type=gpu.BufferType.Standard,
            format=gpu.Format.R32_UINT,
            element_count=math.ceil(Budgets.BYTE_SIZE_WORK_QUEUE_POOL / Budgets.BYTE_SIZE_WORK_QUEUE_FORMAT)
        )

        self.b_work_queue_args = gpu.Buffer(
            name="WorkQueueArgs",
            type=gpu.BufferType.Standard,
            format=gpu.Format.RGBA_32_UINT,
            element_count=1
        )

    def create_constant_buffers(self):
        super().create_constant_buffers()

        self.cb_raster_bin = gpu.Buffer(
            name="ConstantBufferRasterBin",
            type=gpu.BufferType.Structured,
            stride=(4 * 4) * 2,
            element_count=1,
            usage=gpu.BufferUsage.Constant
        )

        self.cb_raster_fine = gpu.Buffer(
            name="ConstantBufferRasterFine",
            type=gpu.BufferType.Structured,
            stride=(4 * 4) * 2,
            element_count=1,
            usage=gpu.BufferUsage.Constant
        )

    def update_resolution_dependent_buffers(self, w, h):
        if w <= self.mW and h <= self.mH:
            return

        self.bin_w = math.ceil(w / Budgets.TILE_SIZE_BIN)
        self.bin_h = math.ceil(h / Budgets.TILE_SIZE_BIN)

        super().update_resolution_dependent_buffers(w, h)

        self.b_bin_counters = gpu.Buffer(
            name="BinCountBuffer",
            type=gpu.BufferType.Standard,
            format=gpu.Format.R32_UINT,
            element_count=self.bin_w * self.bin_h
        )

        self.b_bin_min_z = gpu.Buffer(
            name="BinMinZ",
            type=gpu.BufferType.Standard,
            format=gpu.Format.R32_UINT,
            element_count=self.bin_w * self.bin_h
        )

        self.b_bin_max_z = gpu.Buffer(
            name="BinMaxZ",
            type=gpu.BufferType.Standard,
            format=gpu.Format.R32_UINT,
            element_count=self.bin_w * self.bin_h
        )

        self.b_prefix_sum_args = PrefixSum.allocate_args(self.bin_w * self.bin_h)

    def update_constant_buffers(self, context):
        context.cmd.begin_marker("Update Constant Buffers")

        super().update_constant_buffers(context)

        context.cmd.upload_resource(
            source=np.array([
                context.segment_count,
                context.w,
                context.h,
                Budgets.TILE_SIZE_BIN,
                self.bin_w,
                self.bin_h,
                context.tesselation_sample_count

            ], dtype='f'),
            destination=self.cb_raster_bin
        )

        context.cmd.upload_resource(
            source=np.array([
                context.w,
                context.h,
                Budgets.TILE_SIZE_BIN,
                self.bin_w,
                self.bin_h,
                context.tesselation_sample_count,
                context.oit_opacity,
                context.oit_overlay
            ], dtype='f'),
            destination=self.cb_raster_fine
        )

        context.cmd.end_marker()

    def clear_buffers(self, context):
        context.cmd.begin_marker("Clear Buffers")

        super().clear_buffers(context)

        Utility.clear_buffer(
            context.cmd,
            0,
            1,
            self.b_bin_records_counter,
            Utility.ClearMode.UINT
        )

        Utility.clear_buffer(
            context.cmd,
            0,
            self.bin_w * self.bin_h,
            self.b_bin_counters,
            Utility.ClearMode.UINT
        )

        Utility.clear_buffer(
            context.cmd,
            (1 << 31) - 1, # Max int
            self.bin_w * self.bin_h,
            self.b_bin_min_z,
            Utility.ClearMode.UINT
        )

        Utility.clear_buffer(
            context.cmd,
            0,
            self.bin_w * self.bin_h,
            self.b_bin_max_z,
            Utility.ClearMode.UINT
        )

        context.cmd.end_marker()

    def raster_bin(self, context):
        context.cmd.begin_marker("BinPass")

        context.cmd.dispatch(
            shader=s_raster_bin_tes if context.tesselation else s_raster_bin,

            constants=[
                self.cb_raster_bin
            ],

            inputs=[
                self.b_segment_output,
                self.b_segment_data,
                self.b_vertex_output,
                self.b_segment_header
            ],

            outputs=[
                self.b_bin_records,
                self.b_bin_records_counter,
                self.b_bin_counters,
                self.b_bin_min_z,
                self.b_bin_max_z
            ],

            x=math.ceil(context.segment_count / Budgets.NUM_LANE_PER_WAVE)
        )

        context.cmd.end_marker()

    def build_work_queue(self, context):

        context.cmd.begin_marker("BuildWorkQueue")

        # 1) Generate offset indices into the global work queue, by performing a prefix sum on the bin counters.
        self.b_bin_offsets = PrefixSum.run(
            context.cmd,
            self.b_bin_counters,
            self.b_prefix_sum_args,
            True,
            self.bin_w * self.bin_h
        )

        # 2) Derive a dispatch launch size from the amount of bin records.
        context.cmd.dispatch(
            shader=s_build_work_queue_args,
            inputs=[
                self.b_bin_records_counter
            ],
            outputs=[
                self.b_work_queue_args
            ],
            x=1
        )

        # 3) Indirectly dispatch the work queue construction.
        context.cmd.dispatch(
            indirect_args=self.b_work_queue_args,
            shader=s_build_work_queue,
            inputs=[
                self.b_bin_offsets,
                self.b_bin_records,
                self.b_bin_records_counter
            ],
            outputs=[
                self.b_work_queue
            ]
        )

        context.cmd.end_marker()

    def raster_fine(self, context):
        context.cmd.begin_marker("FinePass")

        if context.oit:
            shader = s_raster_fine_oit_tes if context.tesselation else s_raster_fine_oit
        else:
            shader = s_raster_fine_tes if context.tesselation else s_raster_fine

        context.cmd.dispatch(
            shader=shader,

            constants=[
                self.cb_raster_fine
            ],

            inputs=[
                self.b_work_queue,
                self.b_bin_offsets,
                self.b_bin_counters,
                self.b_segment_data,
                self.b_vertex_output,
                self.b_bin_min_z,
                self.b_bin_max_z
            ],

            outputs=[
                context.target
            ],

            x=self.bin_w,
            y=self.bin_h
        )

        context.cmd.end_marker()

    def go(self, context):
        context.cmd.begin_marker("Raster (Binned)")

        # 1) Dispatch geometry processing and segment setup stages.
        super().go(context)

        # 2) Binning Stage
        self.raster_bin(context)

        # 3) Work Queue
        self.build_work_queue(context)

        # 4) Fine Stage
        self.raster_fine(context)

        context.cmd.end_marker()


from django.contrib import admin
from django.urls import path,include
from . import views
from django.conf.urls import url

urlpatterns = [
    path('', views.home_page, name='home_page'),
    path('filter/', views.filter_news,name='filter_news'),
    path('saved/',views.saved_news,name='saved_news'),
    path('login/',views.LoginView.as_view(), name="login"),
    path('logout/',views.logout,name="logout"),
    path('signup/', views.signup_view, name="signup"),
    path('sent/', views.activation_sent_view, name="activation_sent"),
    path('activate/<slug:uidb64>/<slug:token>/', views.activate, name='activate'),
    path('preferences/', views.preferences,name='preferences'),
]


# 숫자를 string으로 입력 받음
n = input()
# 자릿수를 확인
length = len(n)
# 왼쪽 자릿수들의 합과 오른쪽 자릿수들의 합
sum_left = 0
sum_right = 0

# 반으로 쪼갠 만큼을 확인하기 위한 변수
cnt = 0
for i in n:
  # 왼쪽 자릿수들의 합 구하기
  if cnt <= length / 2 - 1:
    sum_left += int(i)
    cnt += 1
  else:
    # 오른쪽 자릿수들의 합 구하기
    sum_right += int(i)

if sum_left == sum_right:
  print("LUCKY")
else:
  print("READY")

07.럭키_스트레이트.py


#!/usr/bin/env python

import sys
import json
import urllib
import requests
import threading
import xml.etree.ElementTree as ET

def out(s):
  sys.stdout.write(s + '\n')
  sys.stdout.flush()

def err(s):
  sys.stderr.write(s + '\n')
  sys.stderr.flush()

def get_query():
  query = ""
  for i in range(1, len(sys.argv)):
    if not sys.argv[i].startswith("--"):
      query += sys.argv[i] + " "
  return query.strip()

def get_param(key, default = None):
  for i in range(1, len(sys.argv)):
    if sys.argv[i].startswith("--" + key + "="):
      return sys.argv[i][len(key) + 3:]
  return default

def get_value(json, key, default = None):
  if json == None:
    return default
  return json[key] if key in json else default

def get_media(json_str):
  json = {}
  try:
    json = json.loads(json_str)
  except:
    return Media(None, Metadata({}))
  if "tv" in json:
    return TV(json["tv"], Metadata(json))
  if "movie" in json:
    return Movie(json["movie"], Metadata(json))
  return Media(None, Metadata(json))

def has_array(cmp_array, array):
  size = 0
  if array == None:
    return True
  if cmp_array == None:
    return False
  for val in array:
    if val in cmp_array:
      size += 1
  return len(array) == size

def as_int(val):
  if val == None:
    return None
  try:
    return int(val)
  except:
    return None

def load_async_torrents(torrents, index, count):
  threads = []
  size = len(torrents)
  if index + count > size:
    count = size - index
  for i in range(0, count):
    thread = threading.Thread(target = ZooqleTorrent.load_media, args = (torrents[index + i],))
    thread.daemon = True
    thread.start()
    threads.append(thread)
  for thread in threads:
    thread.join()

class Media:
  CATEGORY = "all"
  def __init__(self, json, metadata):
    self.infos = json
    self.metadata = metadata
    self.name = None
    self.year = None

  def __str__(self):
    return self.name + " - " + str(self.year)

class TV(Media):
  CATEGORY = "TV"
  def __init__(self, json, metadata):
    Media.__init__(self, json, metadata)
    self.name = get_value(json, "episode_name")
    self.season = get_value(json, "season")
    self.episode = get_value(json, "episode")
    self.year = get_value(json, "show_started")

  def __str__(self):
    return self.name + " - S" + str(self.season) + "E" + str(self.episode)

class Movie(Media):
  CATEGORY = "Movies"
  def __init__(self, json, metadata):
    Media.__init__(self, json, metadata)
    self.name = get_value(json, "name")
    self.year = get_value(json, "year")

class Metadata:
  QUALITIES = ["Low", "Med", "Std", "720p", "1080p", "Ultra"]
  def __init__(self, json):
    self.infos = get_value(json, "media_info")
    self.quality = get_value(json, "video_quality")
    self.audios = get_value(self.infos, "audio_lang")
    self.subtitles = get_value(self.infos, "subtitle_lang")
    self.size = get_value(json, "size", 0)

  def __str__(self):
    return self.quality

class ZooqleTorrent:
  NAMESPACE = "https://zooqle.com/xmlns/0.1/index.xmlns"
  def __init__(self, xml_node):
    self.title = None
    self.html_link = None
    self.torrent_link = None
    self.seeders = None
    self.leechers = None
    self.info_hash = None
    self.media = None
    self.__load__(xml_node)

  def load_media(self):
    if self.media != None:
      return self.media
    response = requests.get("https://zooqle.com/api/media/" + self.info_hash)
    self.media = get_media(response.content)
    if response.status_code != 200:
      err("load_media(" + str(self.info_hash) + "): " + str(response.status_code) + " - " + str(response.content))
    return self.media

  def __load__(self, xml_node):
    self.title = xml_node.find("title").text.encode("ascii", "ignore")
    self.html_link = xml_node.find("link").text
    self.torrent_link = xml_node.find("enclosure").attrib["url"]
    self.seeders = int(xml_node.find("{" + ZooqleTorrent.NAMESPACE + "}seeds").text)
    self.leechers = int(xml_node.find("{" + ZooqleTorrent.NAMESPACE + "}peers").text)
    self.info_hash = xml_node.find("{" + ZooqleTorrent.NAMESPACE + "}infoHash").text

  def __str__(self):
    return self.title + " [" + str(self.seeders) + ":" + str(self.leechers) + "]"

class Criteria:
  def __init__(self):
    self.query = get_query()
    self.category = get_param("category", "all")
    self.quality = get_param("quality")
    self.min_quality = get_param("min-quality")
    self.year = as_int(get_param("year"))
    self.audios = get_param("audios")
    if self.audios != None:
      self.audios = self.audios.split(',')
    self.subtitles = get_param("subtitles")
    if self.subtitles != None:
      self.subtitles = self.subtitles.split(',')
    self.season = as_int(get_param("season"))
    self.episode = as_int(get_param("episode"))
    self.seeders = as_int(get_param("seeders", 0))
    self.max_size = as_int(get_param("size"))
    self.count = as_int(get_param("count", 0))
    self.load = as_int(get_param("load", 10))

  def matches(self, torrent):
    is_matching = [
      self.__seeders__,
      self.__max_size__,
      self.__quality__,
      self.__min_quality__,
      self.__year__,
      self.__audios__,
      self.__subtitles__,
      self.__season__,
      self.__episode__
    ]
    for matches in is_matching:
      if not matches(torrent):
        return False
    return True

  def __seeders__(self, torrent):
    return torrent.seeders >= self.seeders

  def __max_size__(self, torrent):
    if self.max_size == None:
      return True
    return torrent.media.metadata.size <= self.max_size

  def __quality__(self, torrent):
    if self.quality == None:
      return True
    return torrent.media.metadata.quality == self.quality

  def __min_quality__(self, torrent):
    if self.min_quality == None:
      return True
    try:
      i = Metadata.QUALITIES.index(self.min_quality)
      j = Metadata.QUALITIES.index(torrent.media.metadata.quality)
      return j >= i
    except:
      return False

  def __year__(self, torrent):
    if self.year == None:
      return True
    return torrent.media.year == self.year

  def __audios__(self, torrent):
    return has_array(torrent.media.metadata.audios, self.audios)

  def __subtitles__(self, torrent):
    return has_array(torrent.media.metadata.subtitles, self.subtitles)

  def __season__(self, torrent):
    if self.season == None:
      return True
    if not isinstance(torrent.media, TV):
      return False
    return torrent.media.season == self.season

  def __episode__(self, torrent):
    if self.episode == None:
      return True
    if not isinstance(torrent.media, TV):
      return False
    return torrent.media.episode == self.episode

def zooqle_search(what, category = "all"):
  page = 1
  torrents = []
  has_torrents = True
  while has_torrents:
    response = requests.get("https://zooqle.com/search?q=" + urllib.quote_plus(what) + "+category%3A" + category + "&s=ns&v=t&sd=d&fmt=rss&pg=" + str(page))
    channel = ET.fromstring(response.content)[0]
    has_torrents = False
    for child in channel:
      if child.tag == "item":
        has_torrents = True
        torrents.append(ZooqleTorrent(child))
    page += 1
  return torrents

def usage():
  out("Usage: " + sys.argv[0] + " query [--category=<category>] [--quality=<quality>] [--min-quality=<quality>] [--year=<year>] [--audios=<audios>] [--subtitles=<subtitles>] [--season=<season>] [--episode=<episode>] [--seeders=<seeders>] [--size=<size>] [--count=<count>] [--load=<load>]")
  out("\tcategory\t'TV', 'Movies'")
  out("\tquality\t\t'Low', 'Med', 'Std', '720p', '1080p', 'Ultra'")
  out("\tmin-quality\tMinimum quality required")
  out("\tyear\t\tRelease year")
  out("\taudios\t\tHas audios as <audio1>,<audio2>,...")
  out("\tsubtitles\tHas subtitles as <sub1>,<sub2>,...")
  out("\tseason\t\tSeason number. Only applies for TV")
  out("\tepisode\t\tEpisode number. Only applies for TV")
  out("\tseeders\t\tMinimum number of seeders")
  out("\tsize\t\tMaximal size in bytes")
  out("\tcount\t\tNumber of results")
  out("\tload\t\tNumber of torrents loaded per iteration")
  out("")
  out("Example: " + sys.argv[0] + " rick and morty --category=TV --min-quality=720p --subtitles=en --season=3 --episode=1 --size=1073741824")

if len(sys.argv) == 1:
  usage()
  sys.exit(1)

count = 0
criteria = Criteria()
torrents = zooqle_search(criteria.query, criteria.category)
for i in range(0, len(torrents)):
  if i % criteria.load == 0:
    load_async_torrents(torrents, i, criteria.load)
  if criteria.matches(torrents[i]):
    out(str(torrents[i]) + "\t" + torrents[i].html_link + "\t" + torrents[i].torrent_link)
    count += 1
    if criteria.count != 0 and count >= criteria.count:
      break


#!/usr/bin/python3
import sys

current_key = None
neighbors = list()
suggestion_count = dict() # suggestion_count[x] shows the number of times x is suggested to current key.

def print_list(l):
    if len(l) == 0:
        return
    print(l.pop(0), end="")
    while len(l) > 0:
        print("," + str(l.pop(0)), end="")

def print_result():
    might_list = list()
    # suggestion_count.items() return all key values in suggestion_count.
    # might_list will have all the "id"s that their count in map is 2 or 3.
    for id, cnt in suggestion_count.items():
        if cnt == 2 or cnt == 3:
            might_list.append(id)
    # we need to exclude the neighbors in might list
    might_list = list(filter(lambda x: x not in neighbors, might_list))
    might_list.sort()

    # probably_list will have all the "id"s that their count in map is 2 or 3.
    probably_list = list()
    for id, cnt in suggestion_count.items():
        if cnt >= 4:
            probably_list.append(id)
    # we need to exclude the neighbors in might list
    probably_list = list(filter(lambda x: x not in neighbors, probably_list))
    probably_list.sort()

    print(current_key + ":", end="")
    if len(might_list) > 0:
        print("Might(", end="")
        print_list(might_list)
        print(")", end="")
    print(" ", end="")
    if len(probably_list) > 0:
        print("Probably(", end="")
        print_list(probably_list)
        print(")", end="")
    print()

for line in sys.stdin:
    key, value = line.split("\t", 1)

    type, id = value.strip().split(",")
    id = int(id)
    # print(key + " -> " + type + " " + id)

    # check if new key is started
    if key != current_key: # new key is started
        if current_key != None:
            print_result()
        current_key = key
        # reset neighbors and suggestion_count
        neighbors = list()
        suggestion_count = dict()

    if type == "n":
        neighbors.append(id)
    if type == "s":
        # if a suggestion happened for "id" for first time, set suggestion_count[id] to 0
        if not id in suggestion_count:
            suggestion_count[id] = 0
        # increase count of suggestion
        suggestion_count[id] = suggestion_count[id] + 1
if current_key != None:
  print_result()


#728. Self Dividing Numbers
#related topics
#math

#first solution
def selfDividingNumbers(left, right):
    lst = []
    for i in range(left, right + 1):
        if '0' not in str(i) and sum([i%int(k) for k in str(i)])==0:
            lst.append(i)
    return lst

print(selfDividingNumbers(1, 22))

leet code/Math/728. Self Dividing Numbers.py


# -*- coding: utf-8 -*-

from datetime import datetime, timedelta
import itertools

from ask.models import Question
from contracts import contract
from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse
from django.core.validators import validate_comma_separated_integer_list
from django.db import models
from django.db.models import Count
from django.apps import apps
from django.template import Context, loader
from signalbox.models import Reply, Answer
from signalbox.models.validators import is_24_hour, only_includes_allowed_fields
from signalbox.utils import incomplete, in_range
# from stats.stats import chisquare
from signalbox.s3 import CustomS3BotoStorage
from yamlfield.fields import YAMLField

User = settings.AUTH_USER_MODEL

class StudySite(models.Model):
    """Location (for multicenter studies) at which participants recruited."""

    name = models.CharField(max_length=500,
                            help_text="""e.g. 'Liverpool', 'London' """)

    class Meta:
        app_label = 'signalbox'

    def __unicode__(self):
        return self.name

class StudyManager(models.Manager):
    def get_by_natural_key(self, slug):
        return self.get(slug=slug)

class Study(models.Model):
    """Details of a study (or substudy or a large trial)."""

    objects = StudyManager()

    def natural_key(self):
        return (self.slug, )

    auto_randomise = models.BooleanField(default=True,
                                         help_text="""If on, then users are automatically added to a study
            condition when a new membership is created.""")

    auto_add_observations = models.BooleanField(default=True,
            help_text="""If on, and auto_randomise is also on, then observations will
            be created when a membership is created. Has no effect if
            auto_randomise is off.""")

    visible = models.BooleanField(default=False,
        help_text="""Controls the display of the study on the /studies/ page.""", db_index=True)

    paused = models.BooleanField(default=False,
        help_text="""If True, pauses sending of signals. Observations missed
        will not be caught up later without manual intervention.""", db_index=True)

    participants = models.ManyToManyField(settings.AUTH_USER_MODEL, through='Membership', )

    slug = models.SlugField(help_text="""A short name
        used to refer to the study in URLs and other places.""")

    name = models.CharField(max_length=200)

    study_email = models.EmailField(blank=False,
                                    help_text="The return email address for all mailings/alerts.")

    twilio_number = models.ForeignKey('twiliobox.TwilioNumber', blank=True, null=True)

    ad_hoc_askers = models.ManyToManyField(
        'ask.Asker', blank=True,
        help_text="""Questionnaires which can be completed ad-hoc.""",
        verbose_name="Questionnaires available ad-hoc")

    ad_hoc_scripts = models.ManyToManyField(
        'signalbox.Script', blank=True,
        help_text="""Scripts which can be initiated by users on an ad-hoc basis.
        IMPORTANT. Scripts selected here will appear on the 'add extra data' tab
        of participants' profile pages, and will display the name of the study,
        and the name of the script along with the username of any related
        membership. BE SURE THESE DO NOT LEAK INFORMATION you would not which
        participants to see.""",
        verbose_name="""scripts allowed on an ad-hoc basis""")

    createifs = models.ManyToManyField(
        'signalbox.ObservationCreator', blank=True,
        help_text="""Rules by which new observations might be made in response
        to participant answers.""", verbose_name="""Rules to create new
        observations based on user responses""")

    blurb = models.TextField(blank=True, null=True, help_text="""Snippet of
        information displayed on the studies listing page.""")

    study_image = models.ImageField(blank=True, null=True,
        storage=CustomS3BotoStorage(),
        upload_to="study/images/")

    briefing = models.TextField(blank=True, null=True,
        help_text="""Key information displayed to participants before joining
        the study.""")

    consent_text = models.TextField(blank=True, null=True,
        help_text="""User must agree to this before entering study.""")

    welcome_text = models.TextField(blank=True, null=True,
        default="""Welcome to the study""",
        help_text="""Text user sees in a message box when they signup for the
        the study. Note his message is not needed if participants will be
        added to studies by the experimenter, rather than through the
        website.""")

    max_redial_attempts = models.IntegerField(default=3,
        help_text='''Only relevant for telephone calls: maximium
        number of times to try and complete the call. Default
        if nothing is specified is 3''')

    redial_delay = models.IntegerField(default=30,
        help_text='''Number of minutes to wait before calling again (mainly for phone calls)''')

    working_day_starts = models.PositiveIntegerField(default=8,
        validators=[is_24_hour], help_text="""Hour (24h clock) at which phone
            calls and texts should start to be made""")

    working_day_ends = models.PositiveIntegerField(default=22,
       validators=[is_24_hour], help_text="""Hour (24h clock) at which phone
            calls and texts should stop.""")

    def in_working_hours(self, hour=None):
        """Return True if we are within the study's working hours.

        Working hours are defined by the `working_day_starts` and
        `working_day_ends` fields."""

        hour = hour or datetime.now().time().hour
        workinghours = list(range(
            self.working_day_starts, self.working_day_ends - 1))
        if hour in workinghours:
            return True
        return False

    show_study_condition_to_user = models.BooleanField(default=False,
        help_text="""If True, the user will be able to see their condition on
        their homepage. Useful primarily for experiments where participants
        signs up with the experimenter present and where blinding is not a
        concern (e.g. where the experimenter carries out one of several
        manipulations).""")

    randomisation_probability = models.DecimalField(
        max_digits=2, decimal_places=1, default=.5,
        help_text="""Indicates the probability that the adaptive algorithm will
        choose weighted randomisation rather than allocation to the group which
        minimises the imbalance (minimisation or adaptive randomisation).
        For example, if set to .5, then deterministic allocation to the
        smallest group will only happen half of the time. To turn off adaptive
        randomisation set to 1.""")

    visible_profile_fields = models.CharField(max_length=200,
        blank=True, null=True, help_text="""Available profile fields
        for this study. Can be any of: {0}, separated by a space.""".format(
        ", ".join(settings.USER_PROFILE_FIELDS)),
        validators=[only_includes_allowed_fields])

    required_profile_fields = models.CharField(max_length=200,
        blank=True, null=True, help_text="""Profile fields which users
        will be forced to complete for this study. Can be any of: {0},
        separated by a space.""".format(", ".join(settings.USER_PROFILE_FIELDS)),
        validators=[only_includes_allowed_fields])

    valid_telephone_country_codes = models.CharField(
        validators=[validate_comma_separated_integer_list],
        max_length=32, default="44",
        help_text="""Valid national dialing codes for participants in this study.""")

    def profile_fields_dict(self):
        vis = self.visible_profile_fields or ""
        req = self.required_profile_fields or ""
        return {
            'visible': set(vis.split() + req.split()),
            'required': req.split()
        }

    def randomised(self):
        """Return the Memberships which have been randomised to a Condition in this Study."""

        return self.membership_set.filter(date_randomised__isnull=False)

    def unrandomised(self):
        """Return a Queryset of participants NOT randomised to a Condition."""

        return self.membership_set.filter(date_randomised__isnull=True)

    def scripts_used_by_this_study(self):
        return set(
            itertools.chain(
                *[i.scripts.all() for i in self.studycondition_set.all()])
        )

    def askers_used_by_this_study(self):
        askers = itertools.chain(*[[i.asker for i in j.scripts.all() if i.asker]
                                   for j in self.studycondition_set.all()])
        return set(askers)

    def questions_used(self):
        """Return all :class:`Question`s used to record an :class:`Answer` for this class:`Study`."""

        answers = Answer.objects.filter(reply__observation__dyad__study=self)
        questions = [i.question for i in answers if i.question]

        return list(set(questions))

    def observations_with_duplicate_replies(self):
        Observation = apps.get_model('signalbox', 'Observation')
        obs = Observation.objects.filter(dyad__study=self)
        dupes = obs.annotate(
            num_replies=Count('reply')).filter(num_replies__gt=1)
        return dupes

    def unresolved_observations_with_duplicate_replies(self):
        obs = self.observations_with_duplicate_replies()
        unresolved = [i for i in obs if i.reply_set.filter(
            is_canonical_reply=True).count() == 0]
        return unresolved

    def observations(self):
        Observation = apps.get_model('signalbox', 'Observation')
        return Observation.objects.filter(dyad__study=self)

    class Meta:
        app_label = 'signalbox'
        ordering = ["name"]
        verbose_name_plural = "studies"

    # @contract
    # def chisq_imbalance(self):
    #     """Returns the chi2 and p value for the imbalance in allocations.

    #     That is, between current membership allocations and the expected
    #     frequencies based on the StudyCondition weights.

    #     :rtype: tuple(float, float)
    #     """
    #     conditions = self.studycondition_set.all()
    #     counts = [i.users.count() for i in conditions]
    #     expected = [i.expected_n() for i in conditions]

    #     return chisquare(counts, expected)

    def get_absolute_url(self):
        return reverse('study', args=(self.pk,))

    def clean(self, *args, **kwargs):

        if self.auto_randomise is False and self.auto_add_observations is True:
            raise ValidationError("""Observations can only be added automatically
                when participants are automatically randomised to a condition.""")

        needtwilio = bool(sum([x.script_type.require_study_ivr_number for x in self.scripts_used_by_this_study()]))
        if needtwilio and not self.twilio_number:
            raise ValidationError("This study uses Scripts which will make calls and require Twilio account details.")

        super(Study, self).clean(*args, **kwargs)

    def __unicode__(self):
        return self.name

class StudyCondition(models.Model):
    '''Groupings of participants within a study.'''
    study = models.ForeignKey('signalbox.Study', blank=True, null=True)
    users = models.ManyToManyField(settings.AUTH_USER_MODEL, through='Membership')
    tag = models.SlugField(max_length=255, default="main")
    display_name = models.CharField(max_length=30, blank=True, null=True,
                                    help_text="""A label for this condition which can be shown to
        the Participant (e.g. a non-descriptive name used to identify a
        condition in an experimental session without breaking a blind.""", )
    weight = models.IntegerField(default=1,
                                 help_text="""Relative weights to allocate users to conditions""")
    scripts = models.ManyToManyField('signalbox.Script', blank=True)
    metadata = YAMLField(blank=True, null=True,
        help_text="""YAML meta data available describing this condition. Can be used on Questionnaires,
        e.g. to conditionally display questions.""")

    def expected_n(self):
        """Return the expected number of participants based on the total currently allocated to the parent study."""
        weights = [i.weight for i in self.study.studycondition_set.all()]
        w = self.weight / sum(weights)
        return self.study.membership_set.count() * w

    class Meta:
        ordering = ['study', 'tag']
        app_label = 'signalbox'

    def __unicode__(self):
        return "%s > %s " % (self.study.name, self.tag)


import win32com.client as w32
from os import path as p
from os import remove
from easygui import msgbox, fileopenbox
import sys

class Debtor:
    codes = {1: "площа Праці",
             10: "Сквер Перемоги",
             100: "Тимчасова вулиця",
             11: "Садова",
             12: "1 Травня",
             13: "Зелена",
             14: "Південна",
             16: "Паркова",
             17: "Спортивна",
             18: "Парусна",
             19: "Віталія Шума",
             2: "проспект Миру",
             20: "Данченко",
             3: "Олександрійська",
             31: "Лазурна",
             4: "Корабельна",
             5: "Хантадзе",
             6: "Шевченко",
             7: "Шкільний",
             8: "Торгова",
             9: "вул. Перемоги",
             99999: "Загальноміські об'єкти"
             }
    streets = {
        'Карабельная': 'Корабельна',
        'Александрийская': 'Олександрійська',
        'Олександр.': 'Олександрійська',
        'дом': ', буд.',
        'проспект Мира': 'пр-т Миру',
        'просп.': 'пр-т ',
        'Ленiна': 'пр-т. Миру',
        '1 Мая': '1 Травня',
        '1Травня': '1 Травня',
        'Парковєая': 'Паркова',
        'Парусная': 'Парусна',
        'Г.Сталiнграду ': 'Парусна',
        'Спортивная': 'Спортивна',
        'ул.': u'Вул. ',
        'Виталия': 'В.',
        'Віталия': 'В.',
        'Данченко': 'Данченка',
        'Торговая': 'Торгова',
        'Победы': 'Перемоги',
        'пер.': 'пров. ',
        'Школьный': 'Шкільний',
        'Шевченко': 'Шевченка',
        'Лазурная': 'Лазурна',
        'площадь': 'площа',
        'Труда': 'Праці',
        '\\': '/'

    }  # и т. д

    ## Блок глобвльных данных

    # *****************************************************************
    # метод открыыает файл Excel с должниками, перекодирует его и возвращает ссылку на объект-книгу
    def openExcelInstance(self, fname):
        try:

            self.Excel = w32.DispatchEx("Excel.Application")
            self.wb = self.Excel.Workbooks.Open(fname)
            self.ws = self.wb.ActiveSheet

        except FileNotFoundError as  E:
            print('Файл с долгами не найден')
            print(E)

            self.wb.Close(False)
            self.Excel.Application.Quit()
            sys.exit(-1)
        else:
            return self.Excel, self.wb, self.ws

    # *****************************************************************

    def get_file_name(self):
        """ Окно выбора файла и его возворат"""
        fname = ''
        fname = fileopenbox("Выберите файл", default="*.xlsx;*.dbf")

        filename, file_extension = p.splitext(fname)

        while len(fname) == 0:
            if (file_extension.upper() not in ('.XLSX', '.DBF')):
                msgbox('Это не XLSX и не DBF- файл', ok_button="ОК", title="Перевірте тип файла!")
                fname = ''
                # exit ( )
        return fname

    # *****************************************************************

    def ReplaceInSheet(self, Excel, wb, ws, dictionary: dict, range: str, invert: bool = False):

        def inv_dict(d: dict) -> dict:  # инвертирует словарь
            d = {v: k for k, v in d.items()}
            return d

        # wb- книга, открытая методом OpenXLS

        if invert:
            dictionary = inv_dict(dictionary)

        column = range.split(':')[0]

        for key in dictionary:
            # dest_code = codes.get(source_code)
            value = dictionary[key]
            try:
                try:
                    founded = ws.Range(range).Find(key)
                    ## print(founded)
                    if founded != None:
                        ws.Columns(column).Replace(key, value, 2, 2, False, True)
                except Exception as E:
                    print(E)
            except AttributeError:
                pass

        # wb.Close ( )
        # Excel.Quit ( )

    # *****************************************************************

    def SaveAndClose(self, path, Excel, wb, ws, filetype="XLSX"):
        # ActiveWorkbook.ExportAsFixedFormat Type:=xlTypePDF FileName:="sales.pdf" Quality:=xlQualityStandard DisplayFileAfterPublish:=True

        fullpath = path + u'Боржники_ред.xlsx'
        try:
            if p.exists(fullpath):
                remove(fullpath)
        except OSError as E:
            print(E)
        finally:
            if filetype == "XLSX":
                wb.SaveAs(fullpath, 51)
            elif filetype == "PDF":
                fullpath = path + u'Боржники_ред.pdf'
                ws.ExportAsFixedFormat(0, fullpath, 0, 0)

            wb.Close()
            Excel.Quit()
        return

    # *****************************************************************

    # *****************************************************
    #
    # def addHeader(self, ws):
    #     import datetime
    #     current_month = datetime.datetime.today().month
    #
    #     month_name = {1: "січня",
    #                   2: "лютого",
    #                   3: "березня",
    #                   4: "квітня",
    #                   5: "травня",
    #                   6: "червня",
    #                   7: "липня",
    #                   8: "серпня",
    #                   9: "вересня",
    #                   10: "жовтня",
    #                   11: "листопада",
    #                   12: "грудня"
    #                   }[current_month]
    #
    #     header_city = "м. Чорноморськ"
    #     header_with_month = "Перелік боржників за послуги з утримання будинків та прибудинкових теріторій" \
    #                         "  станом на 01 {} 2017 р. ".format(month_name)
    #
    #     header_warning = "Усім боржникам потрібно терминово погасити існуючу заборгованість" \
    #                      " для подальшого відповідного надання послуг" \
    #                      " з утримання будинків та прибудинкових територій!"
    #
    #     ws.Range("2:9").Insert(-4142)
    #     # for _ in range(5):
    #     #     # посмотреть и изучить количество аргументов и порядок их передачи!!!
    #     #     ws.Selection.Insert (Shift = -4121, CopyOrigin = 0)
    #
    #     ws.Range("A1:F5").ClearContents()
    #
    #     # ws.Selection.HorizontalAlignment = -4108 #xlCenter
    #     # ws.Selection.VerticalAlignment = -4160 #xlTop
    #     # ws.Selection.WrapText = -1
    #     # ws.Selection.Orientation = 0
    #     # ws.Selection.AddIndent = 0
    #     # ws.Selection.IndentLevel = 0
    #     # ws.Selection.ShrinkToFit = 0
    #     # ws.Selection.ReadingOrder = -5002 #xlContext
    #     # ws.Selection.MergeCells = -1
    #     # ws.Selection.Font.Size = 20
    #     # ws.Selection.Font.Bold = -1
    #     # ws.Selection.Font.Color = -16776961
    #
    #     # ws.Selection.Merge()
    #     ws.Range("A1:F1").Merge()
    #     ws.Range('A1').HorizontalAlignment = -4108  # xlCenter
    #     ws.Range('A1').VerticalAlignment = -4160  # xlTop
    #
    #     ws.Range("A1").Value = header_city
    #     ws.Range('A1').Font.Bold = -1
    #
    #     ws.Range('A3:F4').Merge()
    #     ws.Range('A3').HorizontalAlignment = -4108  # xlCenter
    #     ws.Range('A3').VerticalAlignment = -4160  # xlTop
    #     ws.Range('A3').ShrinkToFit = 0
    #     ws.Range('A3').WrapText = -1
    #     ws.Range('A3').Font.Bold = -1
    #     ws.Range("A3").Value = header_with_month
    #
    #     ws.Range('A5:F7').Merge()
    #     ws.Range('A5').HorizontalAlignment = -4108  # xlCenter
    #     ws.Range('A5').VerticalAlignment = -4160  # xlTop
    #     ws.Range('A5').WrapText = -1
    #     ws.Range('A5').Orientation = 0
    #     ws.Range('A5').AddIndent = 0
    #     ws.Range('A5').IndentLevel = 0
    #     ws.Range('A5').ShrinkToFit = -1
    #     ws.Range('A5').ReadingOrder = -5002  # xlContext
    #     ws.Range('A5').MergeCells = -1
    #     ws.Range('A5').Font.Size = 16
    #     ws.Range('A5').Font.Bold = -1
    #     ws.Range('A5').Font.Color = -16776961
    #     ws.Range("A5").RowHeight = 58.5
    #
    #     ws.Range("A5").Value = header_warning
    #
    #     ws.Range('A10').Font.Bold = -1
    #     ws.Range('B10').Font.Bold = -1
    #     ws.Range('C10').Font.Bold = -1
    #
    #     ws.Range('A10').Value = "Вул., буд."
    #     ws.Range('B10').Value = "кв."
    #     ws.Range('C10').Value = "Борг"
    #     ## Все константі перевести в числовой вид!!!
    # # *****************************************************************

if __name__ == '__main__':
    debtor = Debtor()

    fname = debtor.get_file_name()

    Excel, wb, ws = debtor.openExcelInstance(fname)

  #  debtor.addHeader(ws)
    debtor.ReplaceInSheet(Excel, wb, ws)

    path = p.dirname(fname)

    debtor.SaveAndClose(path, Excel, wb, ws, "PDF")


import pymongo
from flask import Flask, render_template, redirect
from scrape_mars import scrape

app = Flask(__name__)

# Set up MongoDB to store data
mongo = pymongo.MongoClient('mongodb://localhost:27017')
db = mongo.mars_db

# Define collection
mars = db.mars

# App route to scrape
@app.route("/scrape")
def store():
    mars_data = scrape()

    # Store data
    mars.update({}, mars_data, upsert=True)

    return redirect("/", code=302)

@app.route("/")
def main():
    # Read data
    mars_data = mars.find_one()

    print(mars_data)

    return render_template('index.html', data=mars_data)

if __name__ == "__main__":
    app.run(debug=True)


from import_data_with_color import importdata
from network_framework_auto import framework
from Record_result import record_label, record_error
import numpy as np
import tensorflow as tf
import openpyxl

def nn_test(test_option,framework_file):
    # Set model path
    model_load_name = test_option['model_load_name']
    model_load_path = './' + test_option['model_path'] + '/' + model_load_name

    # Set parameters
    data_name = test_option['data_name']
    num_class = test_option['num_class']
    test_batch_size = test_option['batch_size']
    keep_prob = test_option['keep_prob']
    save_label = test_option['save_record']
    record_file_name = test_option['record_name']

    # Import test data
    (feature_test, label_test) = importdata(data_name)

    # Build up network framework
    model = framework(feature_test.shape, num_class, framework_data=framework_file, keep_prob=keep_prob)

    # Initialize saver
    saver = tf.train.Saver()

    # Create optimizer and session
    sess = tf.InteractiveSession()
    tf.global_variables_initializer().run()

    # Load per-trained model
    saver.restore(sess, model_load_path)
    print('Load model =', model_load_path)

    # Initialize records
    if save_label:
        record_labels = np.zeros((label_test.shape[1],label_test.shape[1]))

    # Test
    correct_prediction = tf.equal(tf.argmax(model.y,1),tf.argmax(model.y_pred,1))
    correct_batch = tf.reduce_sum(tf.cast(correct_prediction,tf.float32))
    correct_total = 0.

    # Record errors
    error_list = [['Number', 'Predicted', 'True']]

    for i in range(int(feature_test.shape[0] / test_batch_size) + 1):
        batch_feature = feature_test[test_batch_size * i:min(test_batch_size * (i + 1) - 1, feature_test.shape[0])]
        batch_label = label_test[test_batch_size * i:min(test_batch_size * (i + 1) - 1, label_test.shape[0])]
        correct_batch_get, batch_pred_label = sess.run([correct_batch, model.y_pred], feed_dict={model.x: batch_feature, model.y: batch_label})
        correct_total += correct_batch_get

        # Save label output times
        if save_label:
            record_labels = record_labels + record_label(batch_pred_label, batch_label)
            error_tem = record_error(batch_pred_label, batch_label, i * test_batch_size)
            if not error_tem == []:
                error_list = np.append(error_list, error_tem, axis=0)

    # Calculate accuracy
    accuracy = correct_total / feature_test.shape[0]
    print(error_list.shape)

    # Save model
    if save_label:
        # Open work file
        workbook = openpyxl.load_workbook(filename=record_file_name)

        # Create work sheet
        worksheet = workbook.create_sheet(title='test')

        # Record
        worksheet['A1'] = 'Test data'
        worksheet['B1'] = data_name
        worksheet['A2'] = 'Accuracy'
        worksheet['B2'] = '%04f' % accuracy
        worksheet['A3'] = 'Use model'
        worksheet['B3'] = model_load_name

        # Create work sheet
        worksheet = workbook.create_sheet(title='labels')

        # Record labels
        for i in range(record_labels.shape[0]):
            worksheet.cell(row=i + 2, column=1, value=i + 1)
            worksheet.cell(row=1, column=i + 2, value=i + 1)
            for j in range(record_labels.shape[1]):
                worksheet.cell(row=i + 2, column=j + 2, value=record_labels[i,j])

        # Create work sheet
        worksheet = workbook.create_sheet(title='errors')

        for i in range(error_list.shape[0]):
            worksheet.cell(row=i + 1, column=1, value=error_list[i, 0])
            worksheet.cell(row=i + 1, column=2, value=error_list[i, 1])
            worksheet.cell(row=i + 1, column=3, value=error_list[i, 2])

        # Save record
        workbook.save(filename=record_file_name)

    # Display accuracy
    print("Accuracy = ", accuracy)

    return accuracy


"""
Módulo para la creación de los resultados en forma gráfica
especificamente en forma de tabla para la automatización
de los ERDs
"""
from tkinter import *
import customtkinter as ctk

def TableResults(data_list) -> None:
    """
    Crea una interfaz en forma de tabla con los resultados obtenidos

    Args:
        data_list : Lista que contiene la lista de los datos obtenidos
        ticketName: Nombre del titulo que aparecerá en la pantalla de la UI

    Return:
        None
    """
    ctk.set_appearance_mode("dark")
    ctk.set_default_color_theme("dark-blue")

    total_rows = len(data_list)
    total_columns = len(data_list[0])

    class Table:
        def __init__(self, root):
            for i in range(total_rows):
                for j in range(total_columns):
                    if j in [0, 1, 7]:
                        widthColumns = 115
                    elif j == 3:
                        widthColumns = 100
                    elif j == 4 or j == 2:
                        widthColumns = 230
                    else:
                        widthColumns = 210

                    if data_list[i][j] in ["Date",'Time','Action',"ERD", "Expected Data", "Data", "Data to Write", "Result", "Comments"]:
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13, "bold"), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="white", justify="center")
                    elif data_list[i][j] == "PASS":
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13,'bold'), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="#2DF60D", justify="center")
                    elif data_list[i][j] == "FAIL":
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13,'bold'), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="red", justify="center")
                    else:
                        self.Entry = ctk.CTkEntry(root, width=widthColumns, height=23, corner_radius=2,
                                    font=('Lato',13), bg_color="black", fg_color="#1E1E1E",
                                    border_color="#3E3E3E", border_width=1, text_color="white", justify="center")
                    self.Entry.grid(row=i, column=j)
                    self.Entry.insert(END, data_list[i][j])
                    self.Entry.configure(state="readonly")

    root = ctk.CTk()
    # root.title(ticketName)
    Table(root)
    root.mainloop()


from shardimg.classes.manifest import Manifest
from shardimg.utils.files import FileUtils
from shardimg.utils.command import Command
from shardimg.utils.disks import DiskUtils
from shardimg.utils.log import setup_logging
from shardimg.utils.shards import Shards
import random, string, os
logger = setup_logging()

def build_boot_image(
        manifest: Manifest,
        build_dir: str,
        repo: str,
        manifest_path
):
    print(os.path.abspath(manifest_path))
    FileUtils.create_directory(build_dir)
    FileUtils.create_directory(build_dir+"/buildroot")
    FileUtils.create_directory(build_dir + "/buildroot/proc")
    FileUtils.create_directory(build_dir + "/buildroot/sys")
    FileUtils.create_directory(build_dir + "/buildroot/dev")
    FileUtils.create_directory(build_dir+"/root")
    FileUtils.create_directory(build_dir+"/include")

    FileUtils.copy_file(manifest_path, build_dir+"/include/manifest.json", True)

    logger.info(f"Mount proc, sys and dev in {build_dir}/buildroot")
    DiskUtils.mount(source="/proc", mountpoint=build_dir + "/buildroot/proc", fs="proc")
    DiskUtils.mount(source="/sys", mountpoint=build_dir + "/buildroot/sys", fs="sysfs")
    DiskUtils.mount(source="/dev", mountpoint=build_dir + "/buildroot/dev", options=["bind"])

    packages=["base", "dracut", "btrfs-progs", "busybox", "lvm2", "dmraid", "mdadm", "tpm2-tss", "dash", "binutils", "elfutils", manifest.kernelpackage, "linux-firmware"]

    Shards.install_packages(packages, build_dir+"/buildroot")
    Shards.execute_commands(manifest.commands, build_dir+"/buildroot")

    Shards.execute_commands([f'dracut --no-hostonly-cmdline --no-hostonly --uefi --kver {manifest.kernelversion} /{manifest.kernelname}.unsigned.efi'], build_dir+"/buildroot")

    Command.execute_command(
        command=[
            "chown",
            os.getenv("USER"),
            f"{build_dir}/buildroot/{manifest.kernelname}.unsigned.efi",
        ],
        command_description="Correct permissions for kernel efi",
        crash=True,
        elevated=True
    )

    FileUtils.copy_file(build_dir+f"/buildroot/{manifest.kernelname}.unsigned.efi", build_dir+f"/root/{manifest.kernelname}.unsigned.efi")

    logger.info("Unmounting proc, sys and dev")
    DiskUtils.unmount(mountpoint=build_dir + "/buildroot/proc")
    DiskUtils.unmount(mountpoint=build_dir + "/buildroot/sys")
    DiskUtils.unmount(mountpoint=build_dir + "/buildroot/dev")

    Shards.generate_flatpak_manifest(manifest, build_dir)
    Shards.build_flatpak(manifest, build_dir, repo)


import numpy as np
f = lambda x, k : x
h = f
X = np.array([[0],[2],[3],[4]])
Xind = [0,1,2,3]
meanNoiseX = [0]
Q = np.eye(1)*0.0001
R = np.eye(1)*0.0002**2
numberOfSamples = 4
k=0

z = [4]
_6sqrtr = 6*0.2

newXMean = f(X, k)
X2 = newXMean + np.random.multivariate_normal(meanNoiseX,Q,numberOfSamples)
z_est = h(X2, k)

w = multivariate_normal.logpdf(z_est,z,R)*w
b = w.max()
w = np.exp(w - b)
w = (w/np.sum(w))

ind = np.where(diff>_6sqrtr)[0]
ind2 = np.random.choice(Xind, len(ind), True, 1.0-diff/np.sum(diff))
X2[ind] = X2[ind2]
z_est[ind] = z_est[ind2]
#
#self.w = multivariate_normal.logpdf(z_est,z,self.R)*self.w
#b = self.w.max()
#self.w = np.exp(self.w - b)
#self.w = (self.w/np.sum(self.w))
#return self.X[np.argmax(self.w)], self.w


from scrapy import Spider, Request
from crms.items import CrmsItem
import re

class CrmsSpider(Spider):
    name = 'crms_spider'
    allowed_domains = ['www.g2.com']
    start_urls = ['https://www.g2.com/categories/crm?order=g2_score#product-list']

    def parse(self, response):
        total = int(response.xpath('//div[@class="ws-nw"]/strong/text()').extract_first())
        per_page = len(response.xpath('//div[@class="paper pt-half pb-0 my-1"]'))
        total_pages = total // per_page + 1

        # List comprehension to construct all the urls
        result_urls = ['https://www.g2.com/categories/crm?order=g2_score&page={}#product-list'.format(x) for x in range(1,total_pages+1)]

        # Yield the requests to different search result urls,
        # using parse_result_page function to parse the response.
        for url in result_urls:
            yield Request(url=url, callback=self.parse_result_page)

    def parse_result_page(self, response):
        # We are looking for url of the detail page.
        detail_urls = response.xpath('//div[@class="product-listing__title"]/div/div/a[1]/@href').extract()

        # Yield the requests to the details pages,
        # using parse_detail_page function to parse the response.

        for url in detail_urls:
            yield Request(url='https://www.g2.com' + url, callback=self.parse_detail_page)

    # another method

    def parse_detail_page(self, response):

        CRM = response.xpath('//h4[@class="font-weight-bold color-dark-blue word-wrap-word"]/text()').extract()
        Description = response.xpath('//p[@itemprop="description"]/text()').extract()
        OverallRating = response.xpath('//div[@class="mr-4th border px-4th color-primary font-weight-bold border-radius--low bg-white h6"]/text()').extract()
        TotalReviewCount = response.xpath('//div[@class="as-fe"]/text()').extract()
        Review = response.xpath('//h3[@class="review-list-heading m-0 mb-half word-break-word"]/text()').extract()
        Stars = response.xpath('//span[@itemprop="reviewRating"]/meta[3]/@content').extract()
        Date = response.xpath('//div[@class="time-stamp text-small pl-4th ws-nw"]/span/time/text()').extract()

        item = CrmsItem()
        item['CRM'] = CRM
        item['Description'] = Description
        item['OverallRating'] = OverallRating
        item['TotalReviewCount'] = TotalReviewCount
        item['Review'] = Review
        item['Stars'] = Stars
        item['Date'] = Date

        yield item