#!/usr/bin/env python3

# join_and_reformat.py fügt die Textdateien aus einem Ordner in einer neuen
# Textdatei zusammen.
#
# Die Syntax ist:
#
#     $ ./join_and_reformat.py <Quellordner> <Zielordner>
#
# Das Skript führt folgende Operationen aus:
#
#     - Alle Zeilen werden in eine große Liste geschrieben. Dabei wird
#       aus den Quelldateien nur verwendet, was oberhalb der Leerzeile
#       gefolgt von drei Geviertstrichen steht ('———'). Alles darunter
#       wird ignoriert.
#       Der Name der Archivquelle wird in eckigen Klammern als SOURCE_ID
#       vorangestellt.
#     - Die Zeilen werden zusammengefügt.
#     - Die Newlines (Zeilenumbrüche) werden entfernt, außer die Zeile
#       endet mit einem Punkt, Doppelpunkt etc. In der überwiegenden Anzahl
#       der Fälle wird so ein Absatzende markiert. Ein Absatz entspricht
#       jetzt einer Zeile.
#     - Leerzeilen bleiben erhalten.
#     - Es werden alle Worttrennungen angezeigt. Der User entscheidet in
#       jedem Fall, wie damit zu verfahren ist. In der Regel kann die
#       Trennung einfach entfernt werden. Ausnahmen sind "ck"-Trennungen
#       als "k-k" oder Worte mit Bindestrichen, die in der Quelle zufällig
#       an einem Zeilenende lagen.
#     - Das Ergebnis wird eine neue Textdatei geschrieben.

import sys
import os
import re

# get positional parameters
input_folder = sys.argv[1]
output_folder = sys.argv[2]

print()
print(f'Verwende Textdateien aus: {input_folder}')
print(f'Schreibe Ergebnis nach: {output_folder}')
print()

# get the input files (one file per page)
input_files = []
for file in os.listdir(input_folder):
    if file.endswith('.txt'):
        input_files.append(file)
input_files = sorted(input_files)
print('Folgende Dateien wurden gefunden:')
for file in input_files:
    print(file)

# join input files in list of lines. start by adding the source id.
# ignore additions at the bottom of the page.
joined_lines = []
for file in input_files:
    with open(os.path.join(input_folder, file), 'r') as textfile:
        current_page = []
        source_id = file.split('.')[0]
        current_page.append(f'[SOURCE_ID:{source_id}]\n')
        for line in textfile:
            current_page.append(line)
        for index, line in enumerate(current_page):
            if line.isspace() and current_page[index + 1] == '———\n':
                break
            else:
                joined_lines.append(line)

# make paragraphs by eliminating newlines when appropriate.
paragraphs = []
paragraph = ''
par_ends = ('.\n', ':\n', '?\n', '!\n', ')\n', '"\n')
for index, line in enumerate(joined_lines):
    if line.isspace():
        paragraphs.append('\n')
        paragraph = ''
    elif line.endswith(par_ends) or joined_lines[index + 1].isspace():
        paragraph = ' '.join([paragraph, line])
        paragraphs.append(paragraph.lstrip())   # or one line par starts with " "
        paragraph = ''
    else:
        if len(paragraph) == 0:
            paragraph = line.rstrip()
        else:
            paragraph = ' '.join([paragraph, line.rstrip()])

# unhyphenate based on user input
print()
print('Im Folgenden werden Worttrennungen zusammengefügt. Die Möglichkeiten sind:\n'
      '    [Enter]   Normale Trennungen. Das Trennzeichen wird gelöscht, die beiden\n'
      '              Worthälften zusammengefügt.\n'
      '       B      Es handelt sich um einen Bindestrich, der erhalten bleiben muss.\n'
      '       N      Sonderfall ohne Änderungsbedarf.\n'
      '       M      Sonderfall, der manuell nachkorrigiert werden muss. Das Trenn-\n'
      '              Zeichen wird durch "####" ersetzt.')
for par_index, paragraph in enumerate(paragraphs):
    words = paragraph.split(' ')
    words_to_remove = []
    for word_index, word in enumerate(words):
        next_word_index = word_index + 1
        try:
            next_word = words[next_word_index]
        except IndexError:
            next_word = None
        if re.search('[a-zäüöß]+-$', word):
            userinput_makes_no_sense  = True
            while userinput_makes_no_sense:
                print(f'\n>>>   {word} {next_word}\n')
                action = input('[Enter] = Trennung aufheben, "B" = Bindestrich, '
                               '"N" = nichts tun, "M" = markieren\n> ')
                if action == '':
                    joined_word = word.rstrip('-') + next_word
                    words[word_index] = joined_word
                    words_to_remove.append(next_word_index)
                    userinput_makes_no_sense  = False
                elif action.upper() == 'N':
                    userinput_makes_no_sense  = False
                    break
                elif action.upper() == 'B':
                    joined_word = word + next_word
                    words[word_index] = joined_word
                    words_to_remove.append(next_word_index)
                    userinput_makes_no_sense  = False
                elif action.upper() == 'M':
                    joined_word = word.rstrip('-') + '####' + next_word
                    words[word_index] = joined_word
                    words_to_remove.append(next_word_index)
                    userinput_makes_no_sense  = False
                else:
                    print('Falsche Eingabe. Bitte noch einmal.\n')
    words_to_remove = sorted(words_to_remove, reverse=True)
    for position in words_to_remove:
        del words[position]
    paragraphs[par_index] = ' '.join(words)

# write reformatted paragraphs to output file
source_id = input_folder.split('/')[-1]
output_filename = f'{source_id}_joined.txt'
with open(os.path.join(output_folder, output_filename), 'w') as outfile:
    outfile.writelines(paragraphs)
print(f'Ausgabedatei: {os.path.join(output_folder, output_filename)}')
print('Done.')


# HW test for the Fri3dcamp badge

import uasyncio

import os

from neopixels import *

async def cycle_neopixels():
    cycle = 0
    while True:
        cycle = (cycle + 1)
        # print("cycle {}".format(cycle))
        for i in range(0, len(neopixels)):
            pixel = (cycle+i)%len(neopixels)
            # print("pixel {}".format(pixel))
            if (i == 0):
                # print("i == 0")
                neopixels[pixel] = (255,255,255)
            neopixels[pixel] = (10,10,10)
        neopixels.write()

        await uasyncio.sleep_ms(800)

uasyncio.run(cycle_neopixels())


from abc import ABC

import torch
import torch.nn as nn

class GradReverse(torch.autograd.Function):
    """
    Extension of grad reverse layer
    """
    @staticmethod
    def forward(ctx, x, alpha):
        ctx.alpha = alpha
        return x.view_as(x)

    @staticmethod
    def backward(ctx, grad_output):
        grad_output = grad_output.neg() * ctx.alpha
        return grad_output, None

    def grad_reverse(x, alpha):
        return GradReverse.apply(x, alpha)

class DANN(nn.Module):
    def __init__(self, num_classes=10):
        super(DANN, self).__init__()
        self.feature = nn.Sequential(
            nn.Conv2d(3, 32, 5),
            nn.BatchNorm2d(32),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2),

            nn.Conv2d(32, 48, 5),
            nn.BatchNorm2d(48),
            nn.Dropout2d(),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2),
        )

        self.classifier = nn.Sequential(
            nn.Linear(48 * 4 * 4, 100),
            nn.BatchNorm1d(100),
            nn.ReLU(inplace=True),

            nn.Linear(100, 100),
            nn.BatchNorm1d(100),
            nn.ReLU(inplace=True),

            nn.Linear(100, num_classes),
        )

        self.discriminator = nn.Sequential(
            nn.Linear(48 * 4 * 4, 100),
            nn.BatchNorm1d(100),
            nn.ReLU(inplace=True),
            nn.Linear(100, 2),
        )

    def forward(self, x, alpha):
        feat = self.feature(x)
        feat = feat.view(-1, 48 * 4 * 4)

        class_out = self.classifier(feat)

        domain_out = GradReverse.grad_reverse(feat, alpha)
        domain_out = self.discriminator(domain_out)

        return class_out, domain_out

class DigitClassifier(nn.Module):
    def __init__(self, num_classes=10):
        super(DigitClassifier, self).__init__()
        self.feature = nn.Sequential(
            nn.Conv2d(3, 32, 5),
            nn.BatchNorm2d(32),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2),

            nn.Conv2d(32, 48, 5),
            nn.BatchNorm2d(48),
            nn.Dropout2d(),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2),
        )

        self.classifier = nn.Sequential(
            nn.Linear(48 * 4 * 4, 100),
            nn.BatchNorm1d(100),
            nn.ReLU(inplace=True),

            nn.Linear(100, 100),
            nn.BatchNorm1d(100),
            nn.ReLU(inplace=True),

            nn.Linear(100, num_classes),
        )

    def forward(self, x):
        feat = self.feature(x)
        feat = feat.view(-1, 48 * 4 * 4)

        class_out = self.classifier(feat)

        return class_out

class ExtractedDANN(DANN):
    def __init__(self, num_classes=10):
        super().__init__(num_classes)

    def forward(self, x, alpha):
        feat = self.feature(x)
        feat = feat.view(-1, 48 * 4 * 4)
        return feat

if __name__ == '__main__':
    from torchsummary import summary

    # net = Usps2Svhn()
    # net.to("cuda")
    # summary(net, (3, 28, 28))
    # r = torch.randn((10, 3, 28, 28), device="cuda")
    # a, b = net(r, alpha=0.5)

    # print(out.size())

    # classifier = Classifier()
    # classifier.to("cuda")
    # summary(classifier, (48*4*4,))
    # r = torch.randn((10, 48 * 4 * 4), device="cuda")
    # print(classifier(r).size())
    #
    # dis = Discriminator()
    # dis.to("cuda")
    # # summary(dis, (48*4*4,))
    # r = torch.randn((10, 48 * 4 * 4), device="cuda")
    # print(dis(r, alpha=0.5).size())
    net = DigitClassifier()
    net.to("cuda")
    summary(net, (3, 28, 28))
    r = torch.randn((10, 3, 28, 28), device="cuda")
    print(net(r).size())


import pandas as pd

counties = pd.read_csv("source_data/county_fips.csv", dtype={"state_code": "str", "county_code": "str"})
zctas = pd.read_csv("source_data/zcta_county_xwalk.csv", dtype={"zcta": "str", "county": "str"})

states = counties[['state', 'state_name']].drop_duplicates().set_index('state_name')
states.loc['Alaska'][0]
zctas.drop(['county'], axis=1, inplace=True)
zctas['county_state'] = zctas.apply(lambda x: x['county_name'] + ', ' + states.loc[x['state_name']][0], axis=1)
zctas.drop()
zctas[['zcta', 'county_state']].set_index('zcta').to_json('../data/zcta_county_xwlak.json', orient='columns')


from openerp.osv import fields, orm
from openerp.tools.translate import _

class sale_order(orm.Model):
    _inherit = 'sale.order'

    def _calc_amount_untaxed_all_discounted(self, cr, uid, ids,name, args, context=None):
        if context is None:
            context = {}
        res = {}
        for sale_order in self.browse(cr, uid, ids, context=context):
            amount_untaxed = sale_order.amount_untaxed
            discount = (100 - sale_order.financial_discount_percentage - sale_order.global_discount_percentage) / 100
            amount_untaxed_all_discounted = amount_untaxed * discount
            res[sale_order.id] = amount_untaxed_all_discounted
        return res

    def _calc_amount_tax_all_discounted(self, cr, uid, ids,name, args, context=None):
        if context is None:
            context = {}
        res = {}
        for sale_order in self.browse(cr, uid, ids, context=context):
            amount_tax = sale_order.amount_tax
            discount = (100 - sale_order.financial_discount_percentage - sale_order.global_discount_percentage) / 100
            amount_tax_all_discounted = amount_tax * discount
            res[sale_order.id] = amount_tax_all_discounted
        return res

    def _calc_amount_total_all_discounted(self, cr, uid, ids,name, args, context=None):
        if context is None:
            context = {}
        res = {}
        for sale_order in self.browse(cr, uid, ids, context=context):
            amount_total_all_discounted = sale_order.amount_untaxed_all_discounted + sale_order.amount_tax_all_discounted
            res[sale_order.id] = amount_total_all_discounted
        return res

    def _check_if_all_discount(self, cr, uid, ids,name, args, context=None):
        if context is None:
            context = {}
        res = {}
        for sale_order in self.browse(cr, uid, ids, context=context):
            res[sale_order.id] = False
            discount = False
            financial = False
            for line in sale_order.order_line:
                if line.global_discount == True:
                    discount = True
                if line.financial_discount ==True:
                    financial = True
            if discount == True and financial == True:
                res[sale_order.id] = True
        return res

    _columns = {
            'amount_untaxed_all_discounted' : fields.function(_calc_amount_untaxed_all_discounted,string='Untaxed Amount With All Discount', readonly=True),
            'amount_tax_all_discounted' : fields.function(_calc_amount_tax_all_discounted, string='Taxes With All Discount', readonly=True),
            'amount_total_all_discounted' : fields.function(_calc_amount_total_all_discounted, string='Total With All Discount', readonly=True),
            'all_discount_is_present' : fields.function(_check_if_all_discount, string='All Discount Present', readonly=True, type="boolean"),
    }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:


import os
import argparse
from components.Gerbil import Gerbil

default_ref_name = 'qald 9 plus test'
default_ref_file_path = "datasets/qald9plus/dbpedia/qald_9_plus_test_dbpedia.json"

def main():
    parser = argparse.ArgumentParser(
        description="A script to run gerbil experiments")

    parser.add_argument("--ref_name",
                        type=str,
                        help="name of reference file",
                        default=default_ref_name,
                        required=False)
    parser.add_argument("--ref_file_path",
                        type=str,
                        help="path of reference file",
                        default=default_ref_file_path,
                        required=False)
    parser.add_argument("--experiment_id",
                        type=str,
                        help="Experiment id for which results need to be exported",
                        default=None,
                        required=False)
    parser.add_argument("--exp_setting", type=str,
                        help="experiment setting", required=False)
    parser.add_argument("--pred_path", type=str,
                        help="path of prediction files", required=True)

    args = parser.parse_args()

    gerbil = Gerbil()

    if args.experiment_id:
        gerbil.add_experiment_id(args.experiment_id)
        gerbil.export_results(f"{args.pred_path}/result.csv")
        return

    languages = [f.split(".")[0] for f in os.listdir(args.pred_path) if f.endswith('.json')]
    gerbil.add_ref_file(args.ref_name, args.ref_file_path)
    for lang in languages:
        gerbil.add_pred_file(f"{args.exp_setting}-{lang}", f"{args.pred_path}/{lang}.json", lang)
    gerbil.submit_experiment()
    gerbil.export_results(f"{args.pred_path}/result.csv")

if __name__ == "__main__":
    main()


def regex(s, pattern, closed):

	if (s == ''):
		result = ''
		for p in pattern:
			result += p
		return result

	elif ord('(') == ord(s[0]):
		pattern.append('')
		return regex(s[1:], pattern, False)

	elif ord(')') == ord(s[0]):
		if closed:
			pattern[-2] += pattern[-1]
			return regex(s[1:], pattern[:-1], True)
		return regex(s[1:], pattern, True)

	elif (ord(s[0]) == ord('*')) or (ord(s[0]) == ord('?')):
		if (closed):
			return regex(s[1:], pattern[:-1], False)
		elif not closed and len(pattern) == 1:
			pattern[-1] = pattern[-1][:-1]
			return regex(s[1:], pattern, False)
		else:
			pattern[-2] += pattern[-1][:-1]
			return regex(s[1:], pattern[:-1], False)

	elif ord(s[0]) == ord('+'):
		pattern[-2] += pattern[-1]
		return regex(s[1:], pattern[:-1], False)

	else:
		pattern[-1] += s[0]
		return regex(s[1:], pattern, False)

s = input()
#print(regex(s, [''], False))

contexts = ['']
output = ''
closed = False

while s != '':
	c = s[0]

	if ord(c) == ord('('):
		contexts.append('')

	elif ord(c) == ord(')'):
		if closed:
			contexts[-2] += contexts[-1]
			contexts = contexts[:-1]
		closed = True

	elif ord(c) == ord('*') or ord(c) == ord('?'):

		if not closed and len(contexts) > 1:
			contexts[-2] += contexts[-1][:-1]

		elif not closed:
			contexts[-1] = contexts[-1][:-1]

		else:
			contexts = contexts[:-1]
		closed = False

	elif ord(c) == ord('+'):
		if len(contexts) > 1:
			contexts[-2] += contexts[-1]
		else:
			contexts[0] += c

		closed = False

	else:
		if closed:
			contexts[-2] += contexts[-1]
			contexts = contexts[:-1]

		contexts[-1] += c
		closed = False

	s = s[1:]

print(contexts)


import cv2
import GazeFollow
import numpy as np

class FaceDetector:
	"""
	Class to detect faces in picture.
	"""
	def __init__(self, img="face.jpg", debug=False):
		self.face_cascade = cv2.CascadeClassifier('./all_data/haarcascade_frontalface_default.xml')
		self.profile_face_cascade = cv2.CascadeClassifier('./all_data/haarcascade_profileface.xml')
		self.eye_cascade = cv2.CascadeClassifier('./all_data/haarcascade_eye.xml')
		self.img = None
		self.gray = None
		self.loadIm(img)
		if debug:
			self.drawFaceBoxes()
			self.drawCenterFaces()
			self.drawEyeBoxes()
			self.showIm()

	def detectFaces(self):
		"""
		Detects faces in an image.
		Input:
		img = cv.imread image Image in which you want to detect faces.
		Output:
		faces = [(topLeft_x, topLeft_y, width, height)] List of quadrupples that describe a box around the faces that were detected.
		"""
		faces = self.face_cascade.detectMultiScale(self.gray, 1.3, 5)
		profile_faces = self.profile_face_cascade.detectMultiScale(self.gray, 1.3, 5)
		return faces, profile_faces

	def drawFaceBoxes(self):
		faces, prof_faces = self.detectFaces()
		for (x, y, w, h) in faces:
			cv2.rectangle(self.img, (x, y), (x+w, y+h), (255, 0, 0), 2)
		for (x, y, w, h) in prof_faces:
			cv2.rectangle(self.img, (x, y), (x+w, y+h), (255, 0, 0), 2)

	def detectCenterFaces(self):
		centers = []
		height, width, channels = self.img.shape

		faces, prof_faces = self.detectFaces()
		for (x, y, w, h) in faces:
			centers.append([(x + float(w)/2)/width, (y + float(h)/2)/height])
		for (x, y, w, h) in prof_faces:
			centers.append([(x + float(w)/2)/width, (y + float(h)/2)/height])

		return centers

	def drawCenterFaces(self):
		height, width, channels = self.img.shape
		for (x, y) in self.detectCenterFaces():
			cv2.line(self.img, (int(x*width)-5, int(y*height)), (int(x*width)+5, int(y*height)), (255,0,0))
			cv2.line(self.img, (int(x*width), int(y*height)-5), (int(x*width), int(y*height)+5), (255,0,0))

	def detectEyes(self):
		"""
		Detects eyes in an image.
		Input:
		img = cv.imread image Image in which you want to detect faces.
		Output:
		eyes = [(topLeft_x, topLeft_y, width, height)] List of quadrupples that describe a box around the eyes that were detected.
		"""

		faces, prof_faces = self.detectFaces()
		eyes = prof_eyes = []
		for (x, y, w, h) in faces:
			roi_gray = self.gray[y:y+h, x:x+w]
			roi_color = self.img[y:y+h, x:x+w]
			eyes = self.eye_cascade.detectMultiScale(roi_gray)

		for (x, y, w, h) in prof_faces:
			roi_gray = self.gray[y:y+h, x:x+w]
			roi_color = self.img[y:y+h, x:x+w]
			prof_eyes = self.eye_cascade.detectMultiScale(roi_gray)

		return eyes, prof_eyes

	def drawEyeBoxes(self):
		faces, prof_faces = self.detectFaces()
		eyes, prof_eyes = self.detectEyes()

		for (x, y, w, h) in faces:
			for (ex, ey, ew, eh) in eyes:
				roi_color = self.img[y:y+h, x:x+w]
				cv2.rectangle(roi_color, (ex, ey), (ex+ew, ey+eh), (0, 255, 0), 2)
		for (x, y, w, h) in prof_faces:
			for (ex, ey, ew, eh) in prof_eyes:
				roi_color = self.img[y:y+h, x:x+w]
				cv2.rectangle(roi_color, (ex, ey), (ex+ew, ey+eh), (0, 255, 0), 2)

	def loadIm(self, img):
		if isinstance(img, str):
			self.img = cv2.imread(img)
		else:
			self.img = img
		self.gray = cv2.cvtColor(self.img, cv2.COLOR_BGR2GRAY)

	def showIm(self):
		cv2.namedWindow('img', 0)
		cv2.resizeWindow('img', 500, 500)
		cv2.imshow('img', self.img)
		cv2.waitKey(0)
		cv2.destroyAllWindows()

if __name__ == "__main__":
	img = "./all_data/Gaze.jpg"
	fd = FaceDetector(img, True)
	e = fd.detectCenterFaces()
	GN = GazeFollow.GazeNet()
	GN.loadWeights("./all_data/train_GazeFollow/binary_w.npz")
	x, y = GN.getGaze(e[0], img)
	image = cv2.imread(img)
	cv2.line(image, (x - 5, y), (x + 5, y), (255, 0, 0))
	cv2.line(image, (x, y - 5), (x, y + 5), (255, 0, 0))
	cv2.imshow('img', image)
	cv2.waitKey(0)
	cv2.destroyAllWindows()


# run with: python3 seed.py

from dotenv import load_dotenv
from sqlalchemy import inspect
from app import init_app, db
from app.models import AssociationMatchUser, Match, User

load_dotenv()
app = init_app("ProdConfig")
app.app_context().push()
inspector = inspect(db.engine)

# Perfomed because these tables should not typically be dropped,
# and the foreign key dependencies require they be dropped in
# this order, rather than using db.drop_all()
if inspector.has_table(AssociationMatchUser.__table__):
    AssociationMatchUser.__table__.drop(db.engine)
if inspector.has_table(Match.__table__):
    Match.__table__.drop(db.engine)
if inspector.has_table(User.__table__):
    User.__table__.drop(db.engine)

db.create_all()


from __future__ import division
import copy
import datetime
import itertools
import sys

import bson.code
import bson.objectid
from oslo.utils import timeutils
import pymongo
import six

import ceilometer
from ceilometer.openstack.common import log
from ceilometer import storage
from ceilometer.storage import base
from ceilometer.storage import models
from ceilometer.storage.mongo import utils as pymongo_utils
from ceilometer.storage import pymongo_base
from ceilometer import utils

LOG = log.getLogger(__name__)

AVAILABLE_CAPABILITIES = {
    'resources': {'query': {'simple': True,
                            'metadata': True}},
    'statistics': {'groupby': True,
                   'query': {'simple': True,
                             'metadata': True},
                   'aggregation': {'standard': True}}
}

class Connection(pymongo_base.Connection):
    """The db2 storage for Ceilometer

    Collections::

        - meter
          - the raw incoming data
        - resource
          - the metadata for resources
          - { _id: uuid of resource,
              metadata: metadata dictionaries
              user_id: uuid
              project_id: uuid
              meter: [ array of {counter_name: string, counter_type: string,
                                 counter_unit: string} ]
            }
    """

    CAPABILITIES = utils.update_nested(pymongo_base.Connection.CAPABILITIES,
                                       AVAILABLE_CAPABILITIES)
    CONNECTION_POOL = pymongo_utils.ConnectionPool()

    GROUP = {'_id': '$counter_name',
             'unit': {'$min': '$counter_unit'},
             'min': {'$min': '$counter_volume'},
             'max': {'$max': '$counter_volume'},
             'sum': {'$sum': '$counter_volume'},
             'count': {'$sum': 1},
             'duration_start': {'$min': '$timestamp'},
             'duration_end': {'$max': '$timestamp'},
             }

    PROJECT = {'_id': 0, 'unit': 1,
               'min': 1, 'max': 1, 'sum': 1, 'count': 1,
               'avg': {'$divide': ['$sum', '$count']},
               'duration_start': 1,
               'duration_end': 1,
               }

    SORT_OPERATION_MAP = {'desc': pymongo.DESCENDING, 'asc': pymongo.ASCENDING}

    SECONDS_IN_A_DAY = 86400

    def __init__(self, url):

        # Since we are using pymongo, even though we are connecting to DB2
        # we still have to make sure that the scheme which used to distinguish
        # db2 driver from mongodb driver be replaced so that pymongo will not
        # produce an exception on the scheme.
        url = url.replace('db2:', 'mongodb:', 1)
        self.conn = self.CONNECTION_POOL.connect(url)

        # Require MongoDB 2.2 to use aggregate(), since we are using mongodb
        # as backend for test, the following code is necessary to make sure
        # that the test wont try aggregate on older mongodb during the test.
        # For db2, the versionArray won't be part of the server_info, so there
        # will not be exception when real db2 gets used as backend.
        server_info = self.conn.server_info()
        if server_info.get('sysInfo'):
            self._using_mongodb = True
        else:
            self._using_mongodb = False

        if self._using_mongodb and server_info.get('versionArray') < [2, 2]:
            raise storage.StorageBadVersion("Need at least MongoDB 2.2")

        connection_options = pymongo.uri_parser.parse_uri(url)
        self.db = getattr(self.conn, connection_options['database'])
        if connection_options.get('username'):
            self.db.authenticate(connection_options['username'],
                                 connection_options['password'])

        self.upgrade()

    @classmethod
    def _build_sort_instructions(cls, sort_keys=None, sort_dir='desc'):
        """Returns a sort_instruction.

        Sort instructions are used in the query to determine what attributes
        to sort on and what direction to use.
        :param q: The query dict passed in.
        :param sort_keys: array of attributes by which results be sorted.
        :param sort_dir: direction in which results be sorted (asc, desc).
        :return: sort parameters
        """
        sort_keys = sort_keys or []
        sort_instructions = []
        _sort_dir = cls.SORT_OPERATION_MAP.get(
            sort_dir, cls.SORT_OPERATION_MAP['desc'])

        for _sort_key in sort_keys:
            _instruction = (_sort_key, _sort_dir)
            sort_instructions.append(_instruction)

        return sort_instructions

    def upgrade(self, version=None):
        # Establish indexes
        #
        # We need variations for user_id vs. project_id because of the
        # way the indexes are stored in b-trees. The user_id and
        # project_id values are usually mutually exclusive in the
        # queries, so the database won't take advantage of an index
        # including both.
        if self.db.resource.index_information() == {}:
            resource_id = str(bson.objectid.ObjectId())
            self.db.resource.insert({'_id': resource_id,
                                     'no_key': resource_id})
            meter_id = str(bson.objectid.ObjectId())
            timestamp = timeutils.utcnow()
            self.db.meter.insert({'_id': meter_id,
                                  'no_key': meter_id,
                                  'timestamp': timestamp})

            self.db.resource.ensure_index([
                ('user_id', pymongo.ASCENDING),
                ('project_id', pymongo.ASCENDING),
                ('source', pymongo.ASCENDING)], name='resource_idx')

            self.db.meter.ensure_index([
                ('resource_id', pymongo.ASCENDING),
                ('user_id', pymongo.ASCENDING),
                ('project_id', pymongo.ASCENDING),
                ('counter_name', pymongo.ASCENDING),
                ('timestamp', pymongo.ASCENDING),
                ('source', pymongo.ASCENDING)], name='meter_idx')

            self.db.meter.ensure_index([('timestamp',
                                         pymongo.DESCENDING)],
                                       name='timestamp_idx')

            self.db.resource.remove({'_id': resource_id})
            self.db.meter.remove({'_id': meter_id})

        # remove API v1 related table
        self.db.user.drop()
        self.db.project.drop()

    def clear(self):
        # db2 does not support drop_database, remove all collections
        for col in ['resource', 'meter']:
            self.db[col].drop()
        # drop_database command does nothing on db2 database since this has
        # not been implemented. However calling this method is important for
        # removal of all the empty dbs created during the test runs since
        # test run is against mongodb on Jenkins
        self.conn.drop_database(self.db)
        self.conn.close()

    def record_metering_data(self, data):
        """Write the data to the backend storage system.

        :param data: a dictionary such as returned by
                     ceilometer.meter.meter_message_from_counter
        """
        # Record the updated resource metadata
        self.db.resource.update(
            {'_id': data['resource_id']},
            {'$set': {'project_id': data['project_id'],
                      'user_id': data['user_id'] or 'null',
                      'metadata': data['resource_metadata'],
                      'source': data['source'],
                      },
             '$addToSet': {'meter': {'counter_name': data['counter_name'],
                                     'counter_type': data['counter_type'],
                                     'counter_unit': data['counter_unit'],
                                     },
                           },
             },
            upsert=True,
        )

        # Record the raw data for the meter. Use a copy so we do not
        # modify a data structure owned by our caller (the driver adds
        # a new key '_id').
        record = copy.copy(data)
        record['recorded_at'] = timeutils.utcnow()
        # Make sure that the data does have field _id which db2 wont add
        # automatically.
        if record.get('_id') is None:
            record['_id'] = str(bson.objectid.ObjectId())
        self.db.meter.insert(record)

    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery=None, resource=None, pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise ceilometer.NotImplementedError('Pagination not implemented')

        metaquery = metaquery or {}

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in six.iteritems(metaquery)))

        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = pymongo_utils.make_timestamp_range(start_timestamp,
                                                          end_timestamp,
                                                          start_timestamp_op,
                                                          end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource', 'timestamp')
        sort_keys.insert(0, 'resource_id')
        sort_instructions = self._build_sort_instructions(sort_keys=sort_keys,
                                                          sort_dir='desc')
        resource = lambda x: x['resource_id']
        meters = self.db.meter.find(q, sort=sort_instructions)
        for resource_id, r_meters in itertools.groupby(meters, key=resource):
            # Because we have to know first/last timestamp, and we need a full
            # list of references to the resource's meters, we need a tuple
            # here.
            r_meters = tuple(r_meters)
            latest_meter = r_meters[0]
            last_ts = latest_meter['timestamp']
            first_ts = r_meters[-1]['timestamp']

            yield models.Resource(resource_id=latest_meter['resource_id'],
                                  project_id=latest_meter['project_id'],
                                  first_sample_timestamp=first_ts,
                                  last_sample_timestamp=last_ts,
                                  source=latest_meter['source'],
                                  user_id=latest_meter['user_id'],
                                  metadata=latest_meter['resource_metadata'])

    def get_meter_statistics(self, sample_filter, period=None, groupby=None,
                             aggregate=None):
        """Return an iterable of models.Statistics instance.

        Items are containing meter statistics described by the query
        parameters. The filter must have a meter value set.
        """
        if (groupby and
                set(groupby) - set(['user_id', 'project_id',
                                    'resource_id', 'source'])):
            raise ceilometer.NotImplementedError(
                "Unable to group by these fields")

        if aggregate:
            raise ceilometer.NotImplementedError(
                'Selectable aggregates not implemented')

        q = pymongo_utils.make_query_from_filter(sample_filter)

        if period:
            if sample_filter.start:
                period_start = sample_filter.start
            else:
                period_start = self.db.meter.find(
                    limit=1, sort=[('timestamp',
                                    pymongo.ASCENDING)])[0]['timestamp']

        if groupby:
            sort_keys = ['counter_name'] + groupby + ['timestamp']
        else:
            sort_keys = ['counter_name', 'timestamp']

        sort_instructions = self._build_sort_instructions(sort_keys=sort_keys,
                                                          sort_dir='asc')
        meters = self.db.meter.find(q, sort=sort_instructions)

        def _group_key(meter):
            # the method to define a key for groupby call
            key = {}
            for y in sort_keys:
                if y == 'timestamp' and period:
                    key[y] = (timeutils.delta_seconds(period_start,
                                                      meter[y]) // period)
                elif y != 'timestamp':
                    key[y] = meter[y]
            return key

        def _to_offset(periods):
            return {'days': (periods * period) // self.SECONDS_IN_A_DAY,
                    'seconds': (periods * period) % self.SECONDS_IN_A_DAY}

        for key, grouped_meters in itertools.groupby(meters, key=_group_key):
            stat = models.Statistics(unit=None,
                                     min=sys.maxint, max=-sys.maxint,
                                     avg=0, sum=0, count=0,
                                     period=0, period_start=0, period_end=0,
                                     duration=0, duration_start=0,
                                     duration_end=0, groupby=None)

            for meter in grouped_meters:
                stat.unit = meter.get('counter_unit', '')
                m_volume = meter.get('counter_volume')
                if stat.min > m_volume:
                    stat.min = m_volume
                if stat.max < m_volume:
                    stat.max = m_volume
                stat.sum += m_volume
                stat.count += 1
                if stat.duration_start == 0:
                    stat.duration_start = meter['timestamp']
                stat.duration_end = meter['timestamp']
                if groupby and not stat.groupby:
                    stat.groupby = {}
                    for group_key in groupby:
                        stat.groupby[group_key] = meter[group_key]

            stat.duration = timeutils.delta_seconds(stat.duration_start,
                                                    stat.duration_end)
            stat.avg = stat.sum / stat.count
            if period:
                stat.period = period
                periods = key.get('timestamp')
                stat.period_start = (period_start +
                                     datetime.
                                     timedelta(**(_to_offset(periods))))
                stat.period_end = (period_start +
                                   datetime.
                                   timedelta(**(_to_offset(periods + 1))))
            else:
                stat.period_start = stat.duration_start
                stat.period_end = stat.duration_end
            yield stat


import argparse
import json
from ml4eft.limits.optimize_ns import Optimize

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-f",
        "--fit_cards",
        required=True,
        type=str,
        help="fit card"
    )

    parser.add_argument(
        "-c",
        "--coeff",
        required=False,
        type=str,
        nargs = "*",
        help="subset of operators to include"
    )

    args = parser.parse_args()

    with open(args.fit_cards) as json_data:
        config = json.load(json_data)

    runner = Optimize(config, coeff=args.coeff)
    runner.run_sampling()


# from collections import namedtuple
from copy import deepcopy
import logging
import numpy as np
import pandas as pd
from pathlib import Path
import re
# from IPython.display import HTML, display
# from inspect import signature

from .constants import *
from .portfolio import Portfolio
from .distributions import Aggregate, Severity
from .spectral import Distortion
from .parser import UnderwritingLexer, UnderwritingParser
from .utilities import (logger_level, round_bucket, Answer,
                        LoggerManager, qd, show_fig, more,
                        parse_note_ex)

logger = logging.getLogger(__name__)

# rejected: immutable
# WriteAnswer = namedtuple('WriteAnswer', ['kind', 'name', 'spec', 'program', 'object'])

class Underwriter(object):
    """
    The ``Underwriter`` class manages the creation of Aggregate and Portfolio objects, and
    maintains a database of standard Severity (curves) and Aggregate (unit or line level) objects
    called the knowledge base.

    - Handles persistence to and from agg files
    - Is interface into program parser
    - Handles safe lookup from the knowledge for parser

    Objects have a kind and a name. The kind is one of 'sev', 'agg' or 'port'. The name is a string.
    They have a representation as a program. When the program is interpreted it produces a dictionary spec
    that can be used to create the object. The static method factory can create any object from the
    (kind, name, spec, program) quartet, though, strictly, program is not needed.

    The underwriter knowledge is stored in a dataframe indexed by kind and name with columns
    spec and program.
    """

    def __init__(self, name='Rory', databases=None, update=False, log2=10, debug=False):
        """
        Create an underwriter object. The underwriter is the interface to the knowledge base
        of the aggregate system. It is the interface to the parser and the interpreter, and
        to the database of curves, portfolios and aggregates.

        :param name: name of underwriter. Defaults to Rory, after Rory Cline, the best underwriter
            I know and a supporter of an analytic approach to underwriting.
        :param databases: name or list of database files to read in on creation. if None: nothing loaded; if
            'default' (installed) or 'site' (user, in ~/aggregate/databases) database \\*.agg files in default or site
            directory are loaded. If 'all' both default and site databases loaded. A string refers to a single database;
            an interable of strings is also valid. See `read_database` for search path.
        :param update: if True, update database files with new objects.
        :param log2: log2 of number of buckets in discrete representation.  10 is 1024 buckets.
        :param debug: if True, print debug messages.
        """

        self.name = name
        self.update = update
        if log2 <= 0:
            raise ValueError(
                'log2 must be > 0. The number of buckets used equals 2**log2.')
        self.log2 = log2
        self.debug = debug
        self._lexer = None
        self._parser = None
        # make sure all database entries are stored; they are read on demand
        self.databases = [] if databases is None else databases
        # stop pyCharm complaining

        # do not read in until needed for faster loading
        self._default_dir = None
        self._site_dir = None
        self._case_dir = None
        self._template_dir = None
        self._knowledge = pd.DataFrame(columns=['kind', 'name', 'spec', 'program'], dtype=object).set_index(
            ['kind', 'name'])

        # ensure description prints correctly. A bit cheaky.
        pd.set_option('display.max_colwidth', 100)

    @property
    def lexer(self):
        if self._lexer is None:
            self._lexer = UnderwritingLexer()
        return self._lexer

    @property
    def parser(self):
        if self._parser is None:
            self._parser = UnderwritingParser(self.safe_lookup, self.debug)
        return self._parser

    @property
    def default_dir(self):
        # default_dir is installed by pip and contain installation files
        if self._default_dir is None:
            self._default_dir = Path(__file__).parent / 'agg'
            self._default_dir.mkdir(parents=True, exist_ok=True)
        return self._default_dir

    @property
    def site_dir(self):
        # site dir is in Users's home directory and stores their files
        if self._site_dir is None:
            self._site_dir = Path.home() / 'aggregate/databases'
            self._site_dir.mkdir(parents=True, exist_ok=True)
        return self._site_dir

    @property
    def case_dir(self):
        # case dir is in Users's home directory and stores their files
        if self._case_dir is None:
            self._case_dir = Path.home() / 'aggregate/cases'
            # check case dir exists
            self._case_dir.mkdir(parents=True, exist_ok=True)
        return self._case_dir

    @property
    def template_dir(self):
        # template dir is in Users's home directory and stores their files
        if self._template_dir is None:
            self._template_dir = self.default_dir.parent / 'templates'
            self._template_dir.mkdir(parents=True, exist_ok=True)
        return self._template_dir

    def read_databases(self):
        if self.databases is None:
            # nothing to do
            self.databases = []
        elif self.databases == 'all':
            self.databases = ['default', 'site']
        elif type(self.databases) == str:
            self.databases = [self.databases]

        if 'default' in self.databases:
            # add all databases in default_dir
            self.databases.remove('default')
            self.databases.extend(self.default_dir.glob('*.agg'))

        if 'site' in self.databases:
            # add all user databases
            self.databases.remove('site')
            self.databases.extend(self.site_dir.glob('*.agg'))

        # actually read databases
        for fn in self.databases:
            self.read_database(fn)

    def read_database(self, fn):
        """
        read database of curves, aggs and portfolios. These can live in the default directory
        that is part of the instalation or ~/aggregate/

        fn can be a string filename, with or without extension. A .agg extension is
        added if there is no suffix. Search path:

        * in the current dir
        * in site_dir (user)
        * in default_dir (installation)

        :param fn: database file name

        """

        p = Path(fn)
        if p.suffix == '':
            p = p.with_suffix('.agg')
        if p.exists():
            db_path = p
        elif (self.site_dir / p).exists():
            db_path = self.site_dir / fn
        elif (self.default_dir / p).exists():
            db_path = self.default_dir / p
        else:
            logger.error(f'Database {fn} not found. Ignoring.')
            return

        try:
            program = db_path.read_text(encoding='utf-8')
        except Exception as e:
            logger.error(
                f'Error reading requested database {db_path.name}. Ignoring.')
        else:
            # read in, parse, save to sev/agg/port dictionaries
            # throw away answer...not creating anything
            # get rid of cosmetic spaces, but keep newline tabs (2 or more spaces)
            program = re.sub('^  +', '\t', program, flags=re.MULTILINE)
            program = re.sub(' +', ' ', program)
            logger.info(f'Reading database {fn}...')
            n = len(self._knowledge)
            self.interpret_program(program)
            n = len(self._knowledge) - n
            logger.info(
                f'Database {fn} read into knowledge, adding {n} entries.')

    def __getitem__(self, item):
        """
        handles self[item]

        item = 'Name' for all objects called Name
        item = ("kind", "name") for object of kind kind called name

        subscriptable: try user portfolios, b/in portfolios, line, severity
        to access specifically use severity or line methods

        :param item:
        :return:
        """
        # much less fancy version:
        if not isinstance(item, (str, tuple)):
            raise ValueError(
                f'item must be a str (name of object) or tuple (kind, name), not {type(item)}.')

        assert self.knowledge is not None

        try:
            if type(item) == str:
                # name == item, any type
                rows = self._knowledge.xs(
                    item, axis=0, level=1, drop_level=False)
            elif type(item) == tuple:
                # return a dataframe row
                rows = self._knowledge.loc[[item]]
        except KeyError:
            raise KeyError(f'Item {item} not found.')
        except TypeError as e:
            # TODO fix this "TypeError: unhashable type: 'slice'"
            raise KeyError(f'getitem TypeError looking for {item}, {e}')
        else:
            if len(rows) == 1:
                kind, name, spec, program = rows.reset_index().iloc[0]
                return Answer(kind=kind, name=name, spec=spec, program=program, object=None)
            else:
                raise KeyError(
                    f'Error: no unique object found matching {item}. Found {len(rows)} objects.')

    def __repr__(self):
        import aggregate
        s = []
        s.append(f'underwriter        {self.name}')
        s.append(f'version            {aggregate.__version__}')
        s.append(f'knowledge          {len(self.knowledge)} programs')
        s.append(f'update             {self.update}')
        for k in ['log2', 'debug']:
            s.append(f'{k:<19s}{getattr(self, k)}')
        s.append(f'validation_eps     {VALIDATION_EPS}')
        sd = self.site_dir.resolve().relative_to(Path.home())
        sd = f'~/{sd}'
        dd = self.default_dir.resolve()
        try:
            dd = dd.relative_to(Path.home())
            dd = f'~/{dd}'
        except ValueError:
            dd = str(dd)
        s.append(f'site dir           {sd}')
        s.append(f'default dir        {dd}')
        s.append('')
        s.append('help')
        s.append('build.knowledge    list of all programs')
        s.append('build.qshow(pat)   show programs matching pattern')
        s.append('build.show(pat)    build and display matching pattern')
        return '\n'.join(s)

    def factory(self, answer):
        """
        Create object of kind from spec, a dictionary.
        Creating from uw obviously needs the uw, so this is NOT a staticmethod!

        :param answer: an Answer class with members kind, name, spec, and program
        :return: creates answer.object
        """

        kind, name, spec, program, obj = answer.values()

        if obj is not None:
            logger.error(
                f'Surprising: obj from Answer not None, type {type(obj)}. It will be overwritten.')

        if kind == 'agg':
            obj = Aggregate(**spec)
            obj.program = program
        elif kind == 'port':
            # Portfolio expects name, agg_list, uw
            # agg list is a list of spects that can be passed to Aggregate
            # need to drop the 'agg', name before the spec that gets returned
            # by the parser. Hence:
            agg_list = [k for i, j, k in spec['spec']]
            obj = Portfolio(name, agg_list, uw=self)
            obj.program = program
        elif kind == 'sev':
            if 'sev_wt' in spec and spec['sev_wt'] != 1:
                logger.log(WL,
                           f'Mixed severity cannot be created, returning spec. You had {spec["sev_wt"]}, expected 1')
                obj = None
            else:
                obj = Severity(**spec)
                # ? set outside...
                obj.program = program
        elif kind == 'distortion':
            obj = Distortion(**spec)
            # ? set outside
            obj.program = program
        else:
            ValueError(f'Cannot build {kind} objects')

        # update the object
        answer['object'] = obj
        return answer

    @property
    def knowledge(self):
        if len(self._knowledge) == 0 and len(self.databases) > 0:
            # knowledge - accounts and line known to the underwriter
            self.read_databases()
        return self._knowledge.sort_index()[['program', 'spec']]

    @property
    def version(self):
        import aggregate
        return aggregate.__version__

    def test_suite(self):
        f = self.default_dir / 'test_suite.agg'
        txt = f.read_text(encoding='utf-8')
        return txt

    @property
    def test_suite_file(self):
        """
        Return the test_suite filename, or None if it does not exist
        """
        f = self.default_dir / 'test_suite.agg'
        if f.exists():
            return f
        else:
            return None

    def write(self, portfolio_program, log2=0, bs=0, update=None, **kwargs):
        """
        Write a natural language program. Write carries out the following steps.

        1. Read in the program and cleans it (e.g. punctuation, parens etc. are
           removed and ignored, replace ; with new line etc.)

        2. Parse line by line to create a dictionary definition of sev, agg or port objects.

        3. Replace sev.name, agg.name and port.name references with their objects.

        4. If update set, update all created objects.

        Sample input::

            port MY_PORTFOLIO
                agg Line1 20  loss 3 x 2 sev gamma 5 cv 0.30 mixed gamma 0.4
                agg Line2 10  claims 3 x 2 sevgamma 12 cv 0.30 mixed gamma 1.2
                agg Line 3100  premium at 0.4 3 x 2 sev 4 @ lognormal 3 cv 0.8 fixed 1

        The indents are required if each agg item appears on a new line.

        See parser for full language spec! See Aggregate class for many examples.

        :param log2:
        :param bs:
        :param portfolio_program:
        :param update: override class default
        :param kwargs: passed to object's update method if ``update==True``
        :return: single created object or dictionary name: object
        """

        # prepare for update
        # what / how to do; little awkward: to make easier for user have to strip named update args
        # out of kwargs
        if update is None:
            update = self.update

        if update is True and log2 == 0:
            log2 = self.log2

        # first see if portfolio_program refers to a built-in object
        try:
            # calls __getitem__
            answer = self[portfolio_program]
        except (LookupError, TypeError):
            logger.debug(
                f'underwriter.write | object not found, processing as a program.')
        else:
            logger.debug(f'underwriter.write | {answer.kind} object found.')
            answer = self.factory(answer)
            if update:
                answer.object.update(log2, bs, **kwargs)
            # rationalize return to be the same as parsed programs
            # TODO test this code
            return [answer]

        # if you fall through to here then the portfolio_program did not refer to a built-in object
        # run the program, get the interpreter return value, the irv, which contains kind/name->spec,program
        irv = self.interpret_program(portfolio_program)
        rv = []
        for answer in irv:
            # create objects and update if needed
            answer = self.factory(answer)
            if answer not in irv:
                logger.error('OK THAT FAILED' * 20)
            if answer.object is not None:
                # this can fail for named mixed severities, which can only
                # be created in context of an agg... that behaviour is
                # useful for named severities though... hence:
                if update:
                    update_method = getattr(answer.object, 'update', None)
                    if update_method is not None:
                        update_method(log2, bs, **kwargs)
            rv.append(answer)

        # report on what has been done
        if rv is None:
            logger.log(WL, f'Program did not contain any output')
        else:
            if len(rv):
                logger.info(f'Program created {len(rv)} objects.')

        # return created objects
        return rv

    def write_from_file(self, file_name, log2=0, bs=0, update=False, **kwargs):
        """
        Read program from file. Delegates to write.

        :param file_name:
        :param log2:
        :param bs:
        :param update:
        :param kwargs:
        :return:
        """
        portfolio_program = Path(file_name).read_text(encoding='utf-8')
        return self.write(portfolio_program, log2=log2, bs=bs, update=update, **kwargs)

    def interpret_program(self, portfolio_program):
        """
        Preprocess and then parse a program one line at a time. Each output is
        stored in the Underwriter's knowledge database. No objects are created.

        Error handling through parser.

        :param portfolio_program:
        :return:
        """
        # Preprocess ---------------------------------------------------------------------
        portfolio_program = self.lexer.preprocess(portfolio_program)

        # create return value list
        rv = []

        # Parse and Postprocess-----------------------------------------------------------
        # self.parser.reset()
        # program_line_dict = {}
        for program_line in portfolio_program:
            logger.debug(program_line)
            # preprocessor only returns lines of length > 0
            try:
                # parser returns the type, name, and spec of the object
                # this is where you can marry up with the program
                kind, name, spec = self.parser.parse(
                    self.lexer.tokenize(program_line))
            except ValueError as e:
                if isinstance(e.args[0], str):
                    logger.error(e)
                    raise e
                else:
                    t = e.args[0].type
                    v = e.args[0].value
                    i = e.args[0].index
                    txt2 = program_line[0:i] + f'>>>' + program_line[i:]
                    logger.error(
                        f'Parse error in input "{txt2}"\nValue {v} of type {t} not expected')
                    raise e
            else:
                # store in uw dictionary and create if needed
                logger.info(
                    f'answer out: {kind} object {name} parsed successfully...adding to knowledge')
                self._knowledge.loc[(kind, name), :] = [spec, program_line]
                rv.append(Answer(kind=kind, name=name, spec=spec,
                          program=program_line, object=None))

        return rv

    def safe_lookup(self, buildinid):
        """
        Lookup buildinid=kind.name in uw to find expected kind and merge safely into self.arg_dict.

        Different from getitem because it splits the item into kind and name and double
        checks you get the expected kind.

        :param buildinid:  a string in kind.name format
        :return:
        """

        # allow for sev.WC.1 name
        kind, *name = buildinid.split('.')
        name = '.'.join(name)
        try:
            # lookup in Underwriter
            answer = self[(kind, name)]
            found_kind, found_name, spec, program, _ = answer.values()
        except LookupError as e:
            logger.error(f'ERROR id {kind}.{name} not found in the knowledge.')
            raise e
        logger.debug(
            f'UnderwritingParser.safe_lookup | retrieved {kind}.{name} as type {found_kind}.{found_name}')
        if found_kind != kind:
            raise ValueError(
                f'Error: type of {name} is  {found_kind}, not expected {kind}')
        # don't want to pass back the original otherwise changes can be reflected in the knowledge
        spec = deepcopy(spec)
        return spec

    @staticmethod
    def logger_level(level):
        """
        Convenience function.

        :param level:
        :return:
        """
        # set logger_level for all aggregate loggers
        logger_level(level)

    def build(self, program, update=None, log2=0, bs=0, recommend_p=RECOMMEND_P, logger_level=None, **kwargs):
        """
        Convenience function to make work easy for the user. Intelligent auto updating.
        Detects discrete distributions and sets ``bs = 1``.

        ``build`` method sets loger level to 30 by default.

        ``__call__`` is set equal to ``build``.

        :param program:
        :param update: build's update
        :param log2: 0 is default: Estimate log2 for discrete and self.log2 for all others. Inupt value over-rides
            and cancels discrete computation (good for large discrete outcomes where bucket happens to be 1.)
        :param bs:
        :param logger_level: temporary log(ger) level for this build
        :param recommend_p: passed to recommend bucket functions. Increase (closer to 1) for thick tailed distributions.
        :param kwargs: passed to update, e.g., padding. Note force_severity=True is applied automatically
        :return: created object(s)
        """

        # automatically puts level back at the end
        if logger_level is not None:
            lm = LoggerManager(logger_level)

        # make stuff
        # write will return a dict with keys (kind, name) and value a WriteAnswer namedtuple
        rv = self.write(program, update=False, force_severity=True)

        if rv is None or len(rv) == 0:
            logger.log(WL, 'build produced no output')
            return None

        if update is None:
            update = self.update

        # in this loop bs_ and log2_ are the values actually used for each update;
        # they do not overwrite the input default values
        for answer in rv:
            if answer.object is None:
                # object not created
                logger.info(f'Object {answer.name} of kind {answer.kind} returned as '
                            'a spec; no further processing.')
            elif isinstance(answer.object, Aggregate) and update is True:
                # try to guess good defaults
                d = answer.spec
                # extract hints from note string
                log2, bs, recommend_p, kwargs = parse_note_ex(
                    d['note'], log2, bs, recommend_p, kwargs)
                if d['sev_name'] == 'dhistogram' and log2 == 0:
                    bs_ = 1
                    # how big?
                    if d['freq_name'] == 'fixed':
                        max_loss = np.max(d['sev_xs']) * d['exp_en']
                    elif d['freq_name'] == 'empirical':
                        max_loss = np.max(d['sev_xs']) * max(d['freq_a'])
                    elif d['freq_name'] == 'bernoulli':
                        # allow for max loss to occur
                        max_loss = np.max(d['sev_xs'])
                    else:
                        # normal approx on count
                        max_loss = np.max(
                            d['sev_xs']) * d['exp_en'] * (1 + 3 * d['exp_en'] ** 0.5)
                    # binaries are 0b111... len-2 * 2 is len - 1
                    log2_ = len(bin(int(max_loss))) - 1
                    logger.info(f'({answer.kind}, {answer.name}): Discrete mode, '
                                'using bs=1 and log2={log2_}')
                else:
                    if log2 == 0:
                        log2_ = self.log2
                    else:
                        log2_ = log2
                    if bs == 0:
                        bs_ = round_bucket(
                            answer.object.recommend_bucket(log2_, p=recommend_p))
                    else:
                        bs_ = bs
                    logger.info(
                        f'({answer.kind}, {answer.name}): Normal mode, using bs={bs_} (1/{1/bs_}) and log2={log2_}')
                try:
                    answer.object.update(
                        log2=log2_, bs=bs_, debug=self.debug, force_severity=True, **kwargs)
                except ZeroDivisionError as e:
                    logger.error(e)
                except AttributeError as e:
                    logger.error(e)
            elif isinstance(answer.object, Severity):
                # there is no updating for severities
                pass
            elif isinstance(answer.object, Portfolio) and update is True:
                d = answer.spec
                # extract hints from note string
                log2, bs, recommend_p, kwargs = parse_note_ex(
                    d['note'], log2, bs, recommend_p, kwargs)
                # figure stuff
                if log2 == -1:
                    log2_ = 13
                elif log2 == 0:
                    log2_ = self.log2
                else:
                    log2_ = log2
                if bs == 0:
                    bs_ = answer.object.best_bucket(log2_, recommend_p)
                else:
                    bs_ = bs
                logger.info(f'updating with {log2_}, bs=1/{1 / bs_}')
                logger.info(
                    f'({answer.kind}, {answer.name}): bs={bs_} and log2={log2_}')
                answer.object.update(log2=log2_, bs=bs_, remove_fuzz=True, force_severity=True,
                                     debug=self.debug, **kwargs)
            elif isinstance(answer.object, Distortion):
                pass
            elif isinstance(answer.object, (Aggregate, Portfolio)) and update is False:
                pass
            else:
                logger.warning(
                    f'Unexpected: output kind is {type(answer.object)}. (expr/number?)')
                pass

        if len(rv) == 1:
            # only one output...just return that
            # retun object if it exists, otherwise the ans namedtuple
            for answer in rv:
                if answer.object is None:
                    return answer
                else:
                    # "automatic validation" here!
                    # if isinstance(answer.object, (Portfolio, Aggregate)):
                    #     rv = answer.object.valid
                    #     if rv == Validation.NOT_UNREASONABLE:
                    #         logger.warning(f'{answer.kind} {answer.name} is not unreasonable')
                    #     else:
                    #         logger.warning(f'WARNING: {answer.kind} {answer.name} FAIL VALIDATION')
                    return answer.object
        else:
            # multiple outputs, see if there is just one portfolio...this is not ideal?!
            ports_found = 0
            for answer in rv:
                if answer.kind == 'port':
                    ports_found += 1
            if ports_found == 1:
                # if only one, it must be answer
                if answer.object is None:
                    return answer
                else:
                    return answer.object
        # in all other cases, return the full list
        return rv

    __call__ = build

    def interpreter_file(self, *, filename='', where=''):
        """
        Run a suite of test programs. For detailed analysis, run_one.
        filename is a string or Path. If a csv it is read into
        a dataframe, with the first column used as index. If it
        is an agg file (e.g. an agg database), it is preprocessed
        to remove comments and replace \\n\\t agg with a space, then
        split on new lines and converted to a dataframe.
        Other file formats are rejected.

        These methods are called interpreter\_... rather than
        interpret\_... because they are for testing and debugging
        the interpreter, not for actually interpreting anything!

        """
        if filename == '':
            filename = Path.home() / 'aggregate/tests/test_suite.csv'
        elif type(filename) == str:
            filename = Path(filename)
        if filename.suffix == '.csv':
            df = pd.read_csv(filename, index_col=0)
        elif filename.suffix == '.agg':
            txt = filename.read_text(encoding='utf-8')
            stxt = re.sub('\n\tagg', ' agg', txt, flags=re.MULTILINE)
            stxt = stxt.split('\n')
            stxt = [i for i in stxt if len(i) and i[0] != '#']
            df = pd.DataFrame(stxt, columns=['program'])
        else:
            raise ValueError(
                f'File suffix must be .csv or .agg, not {filename.suffix}')
        if where != '':
            df = df.loc[df.index.str.match(where)]
        # add One severity if not input
        # if txt.find('sev One dsev [1]') < 0:
        #     logger.info('Adding One to knowledge.')
        self.write('sev One dsev [1]')
        return self._interpreter_work(df.iterrows())

    def interpreter_line(self, program, name='one off', debug=True):
        """
        Interpret single line of code  in debug mode.
        name is index of output
        """

        return self._interpreter_work(iter([(name, program)]), debug=debug)

    def interpreter_list(self, program_list):
        """
        Interpret elements in a list in debug mode.
        """
        return self._interpreter_work(list(enumerate(program_list)), debug=True)

    def run_test_suite(self):
        """
        Run interpreter on the test suite
        """
        df = self.interpreter_file(filename=self.test_suite_file)
        num_errors = df.error.sum()
        if num_errors != 0:
            logger.error('{num_errors} errors in test suite')
        return df

    def _interpreter_work(self, iterable, debug=False):
        """
        Do all the work for the test, allows input to be marshalled into the tester
        in different ways. Unlike production interpret_program, runs one line at a time.
        Each line is preprocessed and then run through a clean parser, and the output
        analyzed.

        Last column, program as input is only changed if the preprocessor changes the program

        :return: DataFrame
        """
        lexer = UnderwritingLexer()
        parser = UnderwritingParser(self.safe_lookup, debug)
        ans = {}
        errs = 0
        no_errs = 0
        # detect non-trivial change
        def f(x, y): return 'same' if x.replace(
            ' ', '') == y.replace(' ', '').replace('\t', '') else y
        for test_name, program in iterable:
            if type(program) != str:
                program_in = program[0]
                program = lexer.preprocess(program_in)
            else:
                program_in = program
                program = lexer.preprocess(program_in)
            err = 0
            if len(program) == 1:
                program = program[0]
                try:
                    # print(program)
                    kind, name, spec = parser.parse(lexer.tokenize(program))
                except (ValueError, TypeError) as e:
                    errs += 1
                    err = 1
                    kind = program.split()[0]
                    # get something to say about the error
                    ea = getattr(e, 'args', None)
                    if ea is not None:
                        # t = getattr(ea[0], 'type', ea[0])
                        # v = getattr(ea[0], 'value', ea[0])
                        i = getattr(ea[0], 'index', 0)
                        if type(i) != int:
                            i = 0
                        # print(i, ea)
                        txt = program[0:i] + f'>>>' + program[i:]
                        name = 'parse error'
                    else:
                        txt = str(e)
                        name = 'other error'
                    spec = txt
                else:
                    no_errs += 1
                ans[test_name] = [kind, err, name, spec,
                                  program, f(program, program_in)]
            elif len(program) > 1:
                logger.info(
                    f'{program_in} preprocesses to {len(program)} lines; not processing.')
                logger.info(program)
                ans[test_name] = ['multiline', err,
                                  None, None, program, program_in]
            else:
                logger.info(
                    f'{program_in} preprocesses to a blank line; ignoring.')
                ans[test_name] = ['blank', err, None, None, program, program_in]

        df_out = pd.DataFrame(ans,
                              index=['kind', 'error', 'name', 'output', 'preprocessed program',
                                     'program']).T
        df_out.index.name = 'index'
        return df_out

    def more(self, regex):
        """
        More information about methods and properties matching regex

        """
        more(self, regex)

    def qlist(self, regex):
        """
        Wrapper for show to just list elements in knowledge that match ``regex``.
        Returns a dataframe.
        """
        return self.show(regex, kind='', plot=False, describe=False, verbose=True)

    def qshow(self, regex):
        """
        Wrapper for show to just show (display) elements in knowledge that match ``regex``.
        No reutrn value.
        """
        def ff(x):
            fs = '{x:120s}'
            return fs.format(x=x)
        bit = self.show(regex, kind='', plot=False,
                        describe=False)[['program']]
        bit['program'] = bit['program'].str.replace(
            r' note\{[^}]+\}', '').str.replace('  +', ' ')  # , flags=re.MULTILINE)
        # bit['program'] = bit['program'].str.replace(' ( +)', ' ') #, flags=re.MULTILINE)
        # bit['program'] = bit['program'].str.replace(r' note\{[^}]+\}$|  *', ' '   ) #, flags=re.MULTILINE)
        qd(bit,
           line_width=160, max_colwidth=130, col_space=15, justify='left',
           max_rows=200, formatters={'program': ff})

    def show(self, regex, kind='', plot=True, describe=True, logger_level=30, verbose=False, **kwargs):
        """
        Create from knowledge by name or match to name.
        Optionally plot. Returns the created object plus dataframe with more detailed information.
        Allows exploration of preloaded databases.

        Eg ``regex = "A.*[234]`` to run examples named A...2, 3 and 4.

        See ``qshow`` for a wrapper that just returns the matches, with no object
        creation or plotting.

        Examples.
        ::

            from aggregate.utilities import pprint
            # pretty print all prgrams starting A; no object creation
            build.show('^A.*', 'agg', False, False).program.apply(pprint);

            # build and plot A..234
            ans, df = build.show('^A.*')

        :param regex: for filtering name
        :param kind: the kind of object, port, agg, etc.
        :param plot:    if True, plot   (default True)
        :param describe: if True, print the describe dataframe
        :param logger_level: work silently!
        :param verbose: if True, return the dataframe and objects; else no return value
        :param kwargs: passed to build for calculation instructions
        :return: dictionary of created objects and DataFrame with info about each.
        """
        # too painful getting the one thing out!
        ans = []

        # temp logger level
        lm = LoggerManager(logger_level)

        if kind is None or kind == '':
            df = self.knowledge.droplevel('kind').filter(
                regex=regex, axis=0).copy()
        else:
            df = self.knowledge.loc[kind].filter(regex=regex, axis=0).copy()

        # severity causes an error: no est_m etc.
        if "One" in df.index:
            df = df.drop(index='One')

        if plot is False and describe is False:
            # just act like a filtered listing on knowledge
            return df.sort_values('name')

        # added detail columns
        df['log2'] = 0
        df['bs'] = 0.
        df['agg_m'] = 0.
        df['agg_cv'] = 0.
        df['agg_sd'] = 0.
        df['agg_skew'] = 0.
        df['emp_m'] = 0.
        df['emp_cv'] = 0.
        df['emp_sd'] = 0.
        df['emp_skew'] = 0.
        df['valid'] = False

        for n, row in df.iterrows():
            p = row.program
            try:
                a = self.build(p, **kwargs)
                ans.append(a)
            except NotImplementedError:
                logger.error(f'skipping {n}...element not implemented')
            else:
                if describe:
                    pp = getattr(a, 'pprogram', None)
                    if pp is not None:
                        print(pp)
                    qd(a)
                if plot is True:
                    a.plot(figsize=(8, 2.4))
                    # print('\nDensity and Quantiles')
                    print()
                    show_fig(a.figure, format='svg')
                if describe:
                    print('\n')
                df.loc[n, ['log2', 'bs', 'agg_m', 'agg_cv', 'agg_sd', 'agg_skew',
                           'emp_m', 'emp_cv', 'emp_sd', 'emp_skew', 'valid']] = (
                    a.log2, a.bs, a.agg_m, a.agg_cv, a.agg_sd, a.agg_skew, a.est_m, a.est_cv, a.est_sd,
                    a.est_skew, a.explain_validation())
        # if only one item, return it...much easier to use
        if len(ans) == 1:
            # noinspection PyUnboundLocalVariable
            ans = a
        if verbose:
            return ans, df

    def dir(self, pattern=''):
        """
        List all agg databases in site and default directories.
        If entries is True then read them and return named objects.

        :param pattern:  glob pattern for filename; .agg is added

        """

        if pattern == '':
            pattern = '*.agg'
        else:
            pattern += '.agg'

        entries = []

        for dn, d in zip(['site', 'default'], [self.site_dir, self.default_dir]):
            for fn in d.glob(pattern):
                txt = fn.read_text(encoding='utf-8')
                stxt = txt.split('\n')
                for r in stxt:
                    rs = r.split(' ')
                    if rs[0] in ['agg', 'port', 'dist', 'distortion', 'sev']:
                        entries.append([dn, fn.name] + rs[:2])

        ans = pd.DataFrame(entries, columns=[
                           'Directory', 'Database', 'kind', 'name'])
        return ans

# exported instance
# self = dbuild = None
logger_level(30)
build = Underwriter(databases='test_suite', update=True, debug=False, log2=16)
# uncomment to create debug build, add to __init__.py
# debug_build = Underwriter(name='Debug', update=True, debug=True, log2=16)


import time
from selenium import webdriver

driver = webdriver.Chrome('/Users/wuzheyou/PycharmProjects/JOE/chromedriver')
driver.get('https://www.google.com.tw/')
time.sleep(6)
search_box = driver.find_element_by_name('q')
search_box.send_keys('python')
time.sleep(6)
driver.quit()


import os

import json

import pymongo
from pymongo import MongoClient

import glob
import traceback

from utile import load_json

env_path = ""
try:
    client = MongoClient("mongodb://localhost:27017/")
    db = client.test_database
    collection = db.test_collection
    collection.find_one()
    manga = db.manga
    print("connection OK")
except Exception:
    print("vérifier si le server mongoDB est lancé")

def create_manga(manga_title):
    """
    add a new manga

    return id manga inserted
    """
    res = manga.insert_one({"mangaTitle": manga_title, "chapiters": []})

    return res.inserted_id

def read_manga(manga_title):
    """
    return :  a dict with the manga id or None if the manga title don't exist
    """

    return manga.find_one({"mangaTitle": manga_title})

def read_manga_with_chapiters_number(manga_reading):
    """
    return : list of chapitersNumbers
    """

    res = manga_reading["chapiters"]

    return [row["number"] for row in res]

def update_chapiter(manga_title, chap_num, chap_title, chap_date, chap_url):

    return manga.update_one(
        {"mangaTitle": manga_title},
        {
            "$push": {
                "chapiters": {
                    "number": chap_num,
                    "title": chap_title,
                    "date": chap_date,
                    "url": chap_url,
                }
            }
        },
    ).matched_count

def controle_update_chapiter(manga_title, chap_num, chap_title, chap_date, chap_url):
    """

    """
    current_manga = read_manga(manga_title)

    if not current_manga:
        create_manga(manga_title)
        print(f"We had a new manga : {manga_title}")
        current_manga = read_manga(manga_title)

    if chap_num in read_manga_with_chapiters_number(current_manga):
        print(f"Chapiter n°{chap_num} of {manga_title} already exist")
        return 0
    else:
        update_chapiter(manga_title, chap_num, chap_title, chap_date, chap_url)
        print(f"{manga_title} had a new chapiter {chap_num}")
        return 1

def delete_manga_id(manga_id):
    """

    """
    try:
        return manga.delete_one(manga_id).deleted_count
    except Exception:

        traceback.print_exc()

def get_all_manga_title():
    """
        return list of titles of all manga
    """
    liste_manga = manga.aggregate(
        [
            {"$project": {"_id": 0, "mangaTitle": 1}},
            {"$match": {"mangaTitle": {"$exists": True}}},
        ]
    )

    return sorted([x["mangaTitle"] for x in list(liste_manga)])

def my_load():

    # load path
    list_json = sorted(glob.glob(f"{env_path}data/json/*.json"), reverse=True)

    if not list_json:
        print("pas de fichier")
        return 0

    data = load_json(list_json[0])

    for row in data:
        controle_update_chapiter(
            row["mangaTitle"],
            row["chapiterNumber"],
            row["chapTitle"],
            row["chapDate"]["$date"],
            row["link"],
        )

if __name__ == "__main__":
    my_load()


# 导入库
import cv2
import time

# 读取图片
gray = cv2.imread('../../images/source/img10.png', 0)
# 调取opencv库自带的OTSU方法
t1 = time.time()
ret, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)
print("using time", time.time()-t1)
# 可视化 灰度图
cv2.imshow("gray image", gray)
# 可视化 阈值分割图
cv2.imwrite("OTSU.png", thresh)
cv2.namedWindow("threshold image", cv2.WINDOW_NORMAL)
cv2.imshow("threshold image", thresh)
cv2.waitKey()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/10/15 17:53
# @Author  : 马克
#  ADB  安卓平台调试桥 是连接安卓手机和PC端的桥梁
#  运行设备的shell（命令行）
#  adb shell screencap ()    获取设备屏幕截图
#  adb pull （）   传文件
#  adb shell input tap(数值 )     点击手机屏幕  数值是坐标

import os   #导入操作系统的模块
from time import sleep    #  从time模块导入sleep函数

def tap_screen(x,y):
    os.system('adb shell input tap {} {}'.format(x,y))

if __name__ == '__main__':
    for i in range(60):
        print(i)
        tap_screen(1408,880)    # 闯关
        print('开始闯关')
        sleep(22)
        print('进入游戏')
        sleep(2)
        # tap_screen(1795,50)     # 自动
        print('自动按钮点击')
        sleep(32)
        tap_screen(1795,40)     # 跳过
        print('跳过')
        sleep(5)
        tap_screen(1795, 40)  # 跳过
        sleep(1)
        print('跳')
        tap_screen(1795, 40)  # 在跳过
        print('继续跳过')
        sleep(5)
        tap_screen(1500,500)
        print('点击屏幕继续')
        sleep(3)
        tap_screen(1521, 942)  # 再次挑战
        sleep(2)

#
# 暗号：AE86  + 自己昵称
# 马克老师微信号：wqqw0804


# Primitive elixir rules which operate at the file level and don't know anything
# about projects or external dependencies or configuration or anything like that

load(":common.bzl", "ElixirLibrary", "elixir_common_attrs")

def _path(f):
    return f.path

def _ebin_dir(f, mapper = _path):
    path = mapper(f)

    # .ez archives must be named e.g. `hex-0.20.1.ez` and must include e.g. `hex-0.20.1` as a top directory
    if f.extension == "ez":
        path = path + "/" + f.basename[0:-3]
    return path + "/ebin"

# Suitable for use with Args.map_each which expects you to return a list
def _loadpath_arg(lib_dir):
    return ["--loadpath", _ebin_dir(lib_dir)]

def _elixir_library_impl(ctx):
    out = ctx.actions.declare_directory("lib/" + ctx.label.name)
    compile_deps_lib_dirs = depset(transitive = [dep[ElixirLibrary].lib_dirs for dep in ctx.attr.compile_deps])

    args = ctx.actions.args()
    args.add_all(compile_deps_lib_dirs, expand_directories = False, map_each = _loadpath_arg)
    args.add("--output-base-dir", _ebin_dir(out))
    args.add_all(ctx.files.srcs)

    ctx.actions.run(
        executable = ctx.executable._elixir_prim_compiler,
        outputs = [out],
        inputs = depset(
            direct = [],
            transitive = [depset(ctx.files.srcs), compile_deps_lib_dirs],
        ),
        progress_message = "elixir_compile {}".format(", ".join([s.basename for s in ctx.files.srcs])),
        arguments = [args],
        tools = [ctx.executable._elixir_prim_compiler],
    )

    return [
        DefaultInfo(
            files = depset([out]),
        ),
        ElixirLibrary(
            lib_dirs = depset(
                direct = [out],
                transitive = [compile_deps_lib_dirs],
            ),
        ),
    ]

elixir_library = rule(
    _elixir_library_impl,
    attrs = dict(elixir_common_attrs.items() + {
        "srcs": attr.label_list(
            allow_files = [".ex"],
            doc = "Source files",
        ),
        "compile_deps": attr.label_list(
            default = [],
            providers = [ElixirLibrary],
            doc = "Libraries that must be on the loadpath to compile this library",
        ),
        "_elixir_prim_compiler": attr.label(
            default = Label("@rules_elixir//impl/tools:elixir_prim_compiler"),
            allow_single_file = True,
            executable = True,
            cfg = "target",
        ),
    }.items()),
    toolchains = [
        "@rules_elixir//impl/toolchains/otp:toolchain_type",
        "@rules_elixir//impl/toolchains/elixir:toolchain_type",
    ],
    doc = "Builds a directory containing .beam files for all modules in the source file(s)",
)

def _elixir_prebuilt_library_impl(ctx):
    libs = depset(ctx.files.lib_dirs)
    return [
        DefaultInfo(
            files = libs,
        ),
        ElixirLibrary(
            lib_dirs = libs,
        ),
    ]

elixir_prebuilt_library = rule(
    _elixir_prebuilt_library_impl,
    attrs = {
        "lib_dirs": attr.label_list(allow_files = True),
    },
)

_elixir_script_attrs = {
    "srcs": attr.label_list(
        allow_files = [".ex", ".exs"],
        doc = "Script source files to interpret",
    ),
    "eval": attr.string_list(
        default = [],
        doc = "Expressions to evaluate with the -e option",
    ),
    # since scripts don't have a separate compilation step, they just have `deps`,
    # unlike elixir_libraries, which must specify compile and runtime deps separately
    "deps": attr.label_list(allow_files = True),
    "data": attr.label_list(
        allow_files = True,
        doc = "Additional files needed by the script -- these will be added to the script's runfiles",
    ),
    "env": attr.string_dict(
        default = {},
        doc = "Environment variables to set before running the script",
    ),
    "elixir_args": attr.string_list(
        default = [],
        doc = "Additional arguments to be passed to elixir itself",
    ),
    "erl_startup": attr.string(
        default = "-s elixir start_cli",
        doc = "Arguments for telling `erl` to start Elixir (or IEx).",
    ),
    "_script_template": attr.label(
        allow_single_file = True,
        default = Label("//:elixir_script.template"),
    ),
    "_runfiles_lib": attr.label(
        default = Label("@bazel_tools//tools/bash/runfiles"),
    ),
}

# Massage the paths that Bazel gives us so that we can build shell scripts that can find their runfiles
def _rlocation(ctx, f):
    s = f.short_path
    if s[0] == "/":
        # Already an absolute path
        return s
    if s[0:3] == "../":
        # Reference to external workspace, rlocation prefers the absolute path
        return "$(rlocation {})".format(s[3:])

    wsname = f.owner.workspace_name
    if wsname == "":
        # Implicit, but helpful to rlocation
        wsname = ctx.workspace_name
    return "$(rlocation {}/{})".format(wsname, s)

def loadpath_element(ctx, f):
    # .ez archives must be named e.g. `hex-0.20.1.ez` and must include e.g. `hex-0.20.1` as a top directory
    path = _rlocation(ctx, f)
    if f.extension == "ez":
        path = path + "/" + f.basename[0:-3]
    return path + "/ebin"

def _elixir_script_impl(ctx):
    elixir_toolchain = ctx.toolchains["@rules_elixir//impl/toolchains/elixir:toolchain_type"].elixirinfo
    erlang_toolchain = ctx.toolchains["@rules_elixir//impl/toolchains/otp:toolchain_type"].otpinfo
    lib_dirs = depset(
        transitive = [
            elixir_toolchain.lib_dirs,
            depset(transitive = [dep[ElixirLibrary].lib_dirs for dep in ctx.attr.deps if ElixirLibrary in dep]),
        ],
    )

    script_runfiles = ctx.runfiles(
        # Our source files
        files = ctx.files.srcs,
        transitive_files = depset(
            transitive = [
                # erlang/OTP itself
                erlang_toolchain.files,
                # The lib dirs for all the ElixirLibrary targets that we depend on (including elixir itself)
                lib_dirs,
                # Any files passed through the `data` parameter.
                depset(transitive = [dep[DefaultInfo].files for dep in ctx.attr.data]),
                # The runfiles of our dependencies.
                depset(transitive = [dep[DefaultInfo].default_runfiles.files for dep in ctx.attr.deps]),
                # The runfiles library itself.
                ctx.attr._runfiles_lib[DefaultInfo].default_runfiles.files,
            ],
        ),
    )
    arg_groups = [
        ["-r {}".format(_rlocation(ctx, f)) for f in ctx.files.srcs],
        ["-e '{}'".format(expr) for expr in ctx.attr.eval],
        [ctx.expand_location(a) for a in ctx.attr.elixir_args],
    ]

    exe = ctx.actions.declare_file("{}_runner.bash".format(ctx.label.name))

    ctx.actions.expand_template(
        template = ctx.file._script_template,
        output = exe,
        substitutions = {
            "%%loadpath%%": " ".join([loadpath_element(ctx, f) for f in lib_dirs.to_list()]),
            "%%erl%%": _rlocation(ctx, erlang_toolchain.erl),
            "%%erl_startup%%": ctx.attr.erl_startup,
            "%%elixir_args%%": " ".join([arg for g in arg_groups for arg in g]),
            "%%env_vars%%": "\n".join([
                "export {}={}".format(var, val)
                for (var, val) in ctx.attr.env.items() + [("LC_ALL", ctx.attr._elixir_locale)]
            ]),
        },
        is_executable = True,
    )

    return [
        DefaultInfo(
            executable = exe,
            runfiles = script_runfiles,
        ),
        ElixirLibrary(
            lib_dirs = lib_dirs,
        ),
    ]

elixir_script = rule(
    _elixir_script_impl,
    attrs = dict(elixir_common_attrs.items() + _elixir_script_attrs.items()),
    executable = True,
    toolchains = [
        "@rules_elixir//impl/toolchains/otp:toolchain_type",
        "@rules_elixir//impl/toolchains/elixir:toolchain_type",
    ],
    doc = "Elixir script, intended for use with `bazel run` -- does not work outside bazel context",
)

elixir_script_test = rule(
    _elixir_script_impl,
    attrs = dict(elixir_common_attrs.items() + _elixir_script_attrs.items()),
    test = True,
    toolchains = [
        "@rules_elixir//impl/toolchains/otp:toolchain_type",
        "@rules_elixir//impl/toolchains/elixir:toolchain_type",
    ],
    doc = "Elixir test; exactly the same as elixir_script but tagged as a test",
)

def elixir_iex(name = None, elixir_args = [], erl_flags = "", **kwargs):
    elixir_script(
        name = name,
        erl_startup = "-user Elixir.IEx.CLI " + erl_flags,
        elixir_args = ["--no-halt"] + elixir_args,
        **kwargs
    )

def elixir_cli_tool(name = None, srcs = [], deps = [], main = None):
    script_deps = []
    gen_lib = name + "__script_compile"
    elixir_library(
        name = gen_lib,
        srcs = srcs,
        compile_deps = deps,
    )

    entrypoint = None
    if main:
        entrypoint = ["{}.main(System.argv)".format(main)]

    elixir_script(
        name = name,
        eval = entrypoint,
        deps = [gen_lib],
        visibility = ["//visibility:public"],
    )