#!/usr/bin/python
#coding:utf-8

import sys, os, time, string, struct, gevent, re, traceback
import gevent.monkey
from cachetraffic_gl import tasks_repqueue
from cachetraffic_gl import sqlconpool_dict
from cachetraffic_gl import sqlconpool_sem
from cachetraffic_gl import sqlconf_dict
from cachetraffic_gl import sqlconf_sem
from gevent.queue import Queue
from gevent.lock import BoundedSemaphore
from cachetraffic_sql import Mysql
from cachetraffic_log import logger

gevent.monkey.patch_all()

class CacheTrafficRepWorker():
    def __init__(self, config, log):
        #日志对象
        self.log = log
        #配置对象
        self.config = config

    def write_historylog(self, line):
        #保存上报失败日志
        try:
            #临时写文件
            tempath = self.config.HISTORYLOGPATH + '/' + 'day_'+ str(time.time()) + '.temp'
            #正式重命名文件
            path    = self.config.HISTORYLOGPATH + '/' + 'day_'+ str(time.time()) + '.back'
            fp = open(tempath, 'w')
            lines = []
            for value in line:
                domain_data, domain, doamin_ip, domain_bandwidth = value[0:4]
                #将上报信息按行保存到文件
                nodes = domain_data+'\t'+domain+'\t'+str(doamin_ip)+'\t'+str(domain_bandwidth)+'\n'
                lines.append(nodes)
            #行写入
            fp.writelines(lines)
            fp.close()
            #写完毕，重命名
            os.rename(tempath, path)
        except:
            info = sys.exc_info()
            traceback.extract_tb(info[2])
            self.log.error('error %s %s %s', info[0], info[1], tempath)
            return 0

    def update_sql(self, line):

        sqlconpool_sem.acquire()
        #获取相关库的链接
        sqlclient,table = sqlconpool_dict.get("qita", None)
        #上传日表
        sql = "INSERT INTO "+ table+"(BW_DATE, BW_DOMAIN, BW_IP, BW_BANDWIDTH) VALUES(%s,%s,%s,%s)"
        if not sqlclient:
            self.log.error("reportwork sqlclient is not set %s", table)
            sqlconpool_sem.release()
            return 0
        else:
            try:
                param = line
                #插入多行数据
                ret = sqlclient.insertMany(sql, param)
                #事务提交
                sqlclient.end('commit')
            except:
                #事务回滚
                info = sys.exc_info()
                traceback.extract_tb(info[2])
                self.log.error("%s %s %s %s", info[0], info[1], sql, str(param))
                sqlconpool_sem.release()
                return 0
        sqlconpool_sem.release()
        return 1

    def handle_tasks(self):
        self.log.debug("upload work service start")
        #等待日表任务队列
        while 1:
            gevent.sleep(0.5)
            #如果处理队列为空， 则不处理
            if not tasks_repqueue.empty():
                #任务信息
                taskinfo    = tasks_repqueue.get()
            else:
                continue

            #上报日志信息
            ret = self.update_sql(taskinfo)
            if not ret:
                #上报失败
                self.write_historylog(taskinfo)
            else:
                self.log.debug("update sql succeed")


def solution(enroll, referral, seller, amount):
    chk = {}
    p = {}
    # p는 돈 받는 거 기록
    for i in enroll:
        if i not in p:
            p[i] = 0
    # 자신을 추천할 사람이 없으면 그냥 지나가고
    # 추천한 사람이 있으면 chk에 자기가 돈을 줄 사람을
    # 기록
    for idx, refer in enumerate(referral):
        if refer == '-':
            continue
        if enroll[idx] not in chk:
            chk[enroll[idx]] = refer

    def recur(name, money):
        nonlocal chk, p
        if money < 1:
            return
        tmp = money // 10
        p[name] += (money - tmp)
        if name not in chk:
            return

        recur(chk[name], tmp)

    # 판매 집계
    for idx, sell in enumerate(seller):
        # 자신이 돈을 준 사람이 없으면 그대로 가짐

        if sell not in chk:
            p[sell] += int((amount[idx] * 100) * 0.9)
        else:
            recur(sell, amount[idx]*100)

    res = []
    for key in p:
        res.append(p[key])

    return res

프로그래머스/lv3/77486. 다단계 칫솔 판매/다단계 칫솔 판매.py

다단계 칫솔 판매.py


from pyspark import SparkContext
from pyspark.sql import SparkSession, types, Row
from datetime import date, datetime
from decimal import *

sc = SparkContext()
spark = SparkSession(sc)

#Raw Data
RawPath="~raw_data_file_path~"
customer= sc.textFile(RawPath + "customer.tbl*")
orders=sc.textFile(RawPath + "orders.tbl*")
lineitem=sc.textFile(RawPath + "lineitem.tbl*")
supplier=sc.textFile(RawPath + "supplier.tbl*")
nation=sc.textFile(RawPath + "nation.tbl*")
region=sc.textFile(RawPath + "region.tbl*")
part=sc.textFile(RawPath + "part.tbl*")

JsonPath="~export_file_path~"

def string_to_date(date_string):
    """Return a `date` object, created from the string ``date_string``."""
    date_values = date_string.split('-')
    return date(year=int(date_values[0]), month=int(date_values[1]), day=int(date_values[2]))

def getCustomerTblSchema(data):
# root
#  |-- c_custkey: integer (nullable = true)
#  |-- c_name: string (nullable = true)
#  |-- c_address: string (nullable = true)
#  |-- c_nationkey: integer (nullable = true)
#  |-- c_phone: string (nullable = true)
#  |-- c_acctbal: decimal(12,2) (nullable = true)
#  |-- c_mktsegment: string (nullable = true)
#  |-- c_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        c_custkey=int(r[0]),
        c_name=r[1],
        c_address=r[2],
        c_nationkey=int(r[3]),
        c_phone=r[4],
        c_acctbal=Decimal(r[5]),
        c_mktsegment=r[6],
        c_comment=r[7])
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
           types.StructField('c_custkey',types.IntegerType(),False)
           ,types.StructField('c_name',types.StringType())
           ,types.StructField('c_address',types.StringType())
           ,types.StructField('c_nationkey',types.IntegerType())
           ,types.StructField('c_phone',types.StringType())
           ,types.StructField('c_acctbal',types.DecimalType(12,2))
           ,types.StructField('c_mktsegment',types.StringType())
           ,types.StructField('c_comment',types.StringType())])
    return row, schema

def getOrdersTblSchema(data):
# root
#  |-- o_orderkey: integer (nullable = true)
#  |-- o_custkey: integer (nullable = true)
#  |-- o_orderstatus: string (nullable = true)
#  |-- o_totalprice: decimal(12,2) (nullable = true)
#  |-- o_orderdate: date (nullable = true)
#  |-- o_orderpriority: string (nullable = true)
#  |-- o_clerk: string (nullable = true)
#  |-- o_shippriority: integer (nullable = true)
#  |-- o_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        o_orderkey=int(r[0]),
        o_custkey=int(r[1]),
        o_orderstatus=r[2],
        o_totalprice=Decimal(r[3]),
        o_orderdate=r[4],
        o_orderpriority=r[5],
        o_clerk=r[6],
        o_shippriority=int(r[7]),
        o_comment=r[8]
            )
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
        types.StructField('o_orderkey',types.IntegerType(),False)
        ,types.StructField('o_custkey',types.IntegerType(),False)
        ,types.StructField('o_orderstatus',types.StringType())
        ,types.StructField('o_totalprice',types.DecimalType(12,2))
        ,types.StructField('o_orderdate',types.StringType())
        ,types.StructField('o_orderpriority',types.StringType())
        ,types.StructField('o_clerk',types.StringType())
        ,types.StructField('o_shippriority',types.IntegerType())
        ,types.StructField('o_comment',types.StringType())
        ])

    return row, schema

def getLineitemTblSchema(data):
# root
#  |-- l_orderkey: integer (nullable = true)
#  |-- l_partkey: integer (nullable = true)
#  |-- l_suppkey: integer (nullable = true)
#  |-- l_linenumber: integer (nullable = true)
#  |-- l_quantity: decimal(12,2) (nullable = true)
#  |-- l_extendedprice: decimal(12,2) (nullable = true)
#  |-- l_discount: decimal(12,2) (nullable = true)
#  |-- l_tax: decimal(12,2) (nullable = true)
#  |-- l_returnflag: string (nullable = true)
#  |-- l_linestatus: string (nullable = true)
#  |-- l_shipdate: date (nullable = true)
#  |-- l_commitdate: date (nullable = true)
#  |-- l_receiptdate: date (nullable = true)
#  |-- l_shipinstruct: string (nullable = true)
#  |-- l_shipmode: string (nullable = true)
#  |-- l_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        l_orderkey=int(r[0]),
        l_partkey=int(r[1]),
        l_suppkey=int(r[2]),
        l_linenumber=int(r[3]),
        l_quantity=Decimal(r[4]),
        l_extendedprice=Decimal(r[5]),
        l_discount=Decimal(r[6]),
        l_tax=Decimal(r[7]),
        l_returnflag=r[8],
        l_linestatus=r[9],
        l_shipdate=r[10],
        l_commitdate=r[11],
        l_receiptdate=r[12],
        l_shipinstruct=r[13],
        l_shipmode=r[14],
        l_comment=r[15]
            )
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
        types.StructField('l_orderkey',types.IntegerType(),False)
        ,types.StructField('l_partkey',types.IntegerType(),False)
        ,types.StructField('l_suppkey',types.IntegerType(),False)
        ,types.StructField('l_linenumber',types.IntegerType())
        ,types.StructField('l_quantity',types.DecimalType(12,2))
        ,types.StructField('l_extendedprice',types.DecimalType(12,2))
        ,types.StructField('l_discount',types.DecimalType(12,2))
        ,types.StructField('l_tax',types.DecimalType(12,2))
        ,types.StructField('l_returnflag',types.StringType())
        ,types.StructField('l_linestatus',types.StringType())
        ,types.StructField('l_shipdate',types.StringType())
        ,types.StructField('l_commitdate',types.StringType())
        ,types.StructField('l_receiptdate',types.StringType())
        ,types.StructField('l_shipinstruct',types.StringType())
        ,types.StructField('l_shipmode',types.StringType())
        ,types.StructField('l_comment',types.StringType())
    ])

    return row, schema

def getSupplierTblSchema(data):
# root
#  |-- s_suppkey: integer (nullable = true)
#  |-- s_name: string (nullable = true)
#  |-- s_address: string (nullable = true)
#  |-- s_nationkey: integer (nullable = true)
#  |-- s_phone: string (nullable = true)
#  |-- s_acctbal: decimal(12,2) (nullable = true)
#  |-- s_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        s_suppkey=int(r[0]),
        s_name=r[1],
        s_address=r[2],
        s_nationkey=int(r[3]),
        s_phone=r[4],
        s_acctbal=Decimal(r[5]),
        s_comment=r[6]
            )
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
        types.StructField('s_suppkey',types.IntegerType(),False)
        ,types.StructField('s_name',types.StringType())
        ,types.StructField('s_address',types.StringType())
        ,types.StructField('s_nationkey',types.IntegerType())
        ,types.StructField('s_phone',types.StringType())
        ,types.StructField('s_acctbal',types.DecimalType(12,2))
        ,types.StructField('s_comment',types.StringType())
        ])

    return row, schema

def getNationTblSchema(data):
# root
#  |-- n_nationkey: integer (nullable = true)
#  |-- n_name: string (nullable = true)
#  |-- n_regionkey: integer (nullable = true)
#  |-- n_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        n_nationkey=int(r[0]),
        n_name=r[1],
        n_regionkey=int(r[2]),
        n_comment=r[3]
            )
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
        types.StructField('n_nationkey',types.IntegerType(),False)
        ,types.StructField('n_name',types.StringType())
        ,types.StructField('n_regionkey',types.IntegerType())
        ,types.StructField('n_comment',types.StringType())
        ])

    return row, schema

def getRegionTblSchema(data):
# root
#  |-- r_regionkey: integer (nullable = true)
#  |-- r_name: string (nullable = true)
#  |-- r_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        r_regionkey=int(r[0]),
        r_name=r[1],
        r_comment=r[2]
            )
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
        types.StructField('r_regionkey',types.IntegerType(),False)
        ,types.StructField('r_name',types.StringType())
        ,types.StructField('r_comment',types.StringType())
        ])

    return row, schema

def getPartTblSchema(data):
# root
#  |-- p_partkey: integer (nullable = true)
#  |-- p_name: string (nullable = true)
#  |-- p_mfgr: string (nullable = true)
#  |-- p_brand: string (nullable = true)
#  |-- p_type: string (nullable = true)
#  |-- p_size: integer (nullable = true)
#  |-- p_container: string (nullable = true)
#  |-- p_retailprice: decimal(12,2) (nullable = true)
#  |-- p_comment: string (nullable = true)

    # map the data, return data as a Row, and cast data types of some fields
    split = data.map(lambda l: l.split("|"))
    row = split.map(lambda r: Row(
        p_partkey=int(r[0]),
        p_name=r[1],
        p_mfgr=r[2],
        p_brand=r[3],
        p_type=r[4],
        p_size=int(r[5]),
        p_container=r[6],
        p_retailprice=Decimal(r[7]),
        p_comment=r[8]
            )
        )

    #build strict schema for the table closely based on the original sql schema
    schema = types.StructType([
        types.StructField('p_partkey',types.IntegerType(),False)
        ,types.StructField('p_name',types.StringType())
        ,types.StructField('p_mfgr',types.StringType())
        ,types.StructField('p_brand',types.StringType())
        ,types.StructField('p_type',types.StringType())
        ,types.StructField('p_size',types.IntegerType())
        ,types.StructField('p_container',types.StringType())
        ,types.StructField('p_retailprice',types.DecimalType(12,2))
        ,types.StructField('p_comment',types.StringType())
        ])

    return row, schema

def createTable(tblName):
    """Create table from raw data and write to json file
    @param tblName string name of the table
    """
    row = None
    schema = None
    if tblName == "customer":
        row, schema = getCustomerTblSchema(customer)
    elif tblName == "orders":
        row, schema = getOrdersTblSchema(orders)
    elif tblName == "lineitem":
        row, schema = getLineitemTblSchema(lineitem)
    elif tblName == "supplier":
        row, schema = getSupplierTblSchema(supplier)
    elif tblName == "nation":
        row, schema = getNationTblSchema(nation)
    elif tblName == "region":
        row, schema = getRegionTblSchema(region)
    elif tblName == "part":
        row, schema = getPartTblSchema(part)
    else:
        return

    # create data frame by passing in the mapped data AND its strict schema
    df = spark.createDataFrame(row, schema)

    #create json file
    df.write.json(JsonPath + tblName + ".json")
    #write to local - for testing purpose
    #df.write.json("./" + tblName + ".json")

    # create temp table name of the new table
    df.createOrReplaceTempView(tblName)
    return

createTable("customer")
createTable("orders")
createTable("lineitem")
createTable("supplier")
createTable("nation")
createTable("region")
createTable("part")

spark.stop()


import os
import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
from torchvision.utils import draw_bounding_boxes
import torchvision.transforms.functional as F

from clearpose.xu_6dof.networks.references.segmentation.train import train_one_epoch
import clearpose.xu_6dof.networks.references.segmentation.transforms as T

from clearpose.xu_6dof.networks.stage1.surface_normals.deeplabv3 import build_model
from clearpose.xu_6dof.datasets.surface_normal_dataset import SurfaceNormalDataset

class NormalCriterion(nn.Module):
	def __init__(self):
		super(NormalCriterion, self).__init__()
		self.cosine_similarity = torch.nn.CosineSimilarity(dim=1)

	def forward(self, output, target):
		norm = torch.linalg.vector_norm(target, dim=1)
		norm_nonzero = torch.nonzero(norm, as_tuple=True)
		output_nonzero = output[norm_nonzero[0],:,norm_nonzero[1],norm_nonzero[2]]
		target_nonzero = target[norm_nonzero[0],:,norm_nonzero[1],norm_nonzero[2]]

		return torch.mean(1-self.cosine_similarity(output_nonzero, target_nonzero))

def get_transform(train):
	transforms = []
	transforms.append(T.PILToTensor())
	transforms.append(T.ConvertImageDtype(torch.float))
	transforms.append(T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))
	if train:
		transforms.append(T.RandomHorizontalFlip(0.5))
	return T.Compose(transforms)

def show(imgs):
	if not isinstance(imgs, list):
		imgs = [imgs]
	fix, axs = plt.subplots(ncols=len(imgs), squeeze=False)
	for i, img in enumerate(imgs):
		img = img.detach()
		img = F.to_pil_image(img)
		axs[0, i].imshow(np.asarray(img))
		axs[0, i].set(xticklabels=[], yticklabels=[], xticks=[], yticks=[])
	plt.show()

def main(save_dir=os.path.join("experiments","xu_6dof","stage1","surface_normals","models")):
	# train on the GPU or on the CPU, if a GPU is not available
	device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

	# use our dataset and defined transformations
	dataset = SurfaceNormalDataset(transforms=get_transform(train=True))
	dataset_test = SurfaceNormalDataset(transforms=get_transform(train=False))

	# split the dataset in train and test set
	indices = torch.randperm(len(dataset)).tolist()
	dataset = torch.utils.data.Subset(dataset, indices[:-50])
	dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])

	# define training and validation data loaders
	data_loader = torch.utils.data.DataLoader(
		dataset, batch_size=2, drop_last=True, shuffle=True, num_workers=4)

	data_loader_test = torch.utils.data.DataLoader(
		dataset_test, batch_size=1, shuffle=False, num_workers=4)

	criterion = NormalCriterion()

	# get the model using our helper function
	model = build_model()

	# move model to the right device
	model.to(device)

	# construct an optimizer
	params = [p for p in model.parameters() if p.requires_grad]
	optimizer = torch.optim.SGD(params, lr=0.005,
								momentum=0.9, weight_decay=0.0005)
	# and a learning rate scheduler
	lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
												   step_size=3,
												   gamma=0.1)

	# let's train it for 10 epochs
	num_epochs = 10

	torch.save(model.state_dict(), os.path.join(save_dir,"deeplabv3_0.pt"))
	for epoch in range(num_epochs):
		# train for one epoch, printing every 10 iterations
		train_one_epoch(model, criterion, optimizer, data_loader, lr_scheduler, device, epoch, print_freq=100)
		# update the learning rate
		torch.save(model.state_dict(), os.path.join(save_dir,"deeplabv3_"+str(epoch)+".pt"))
		lr_scheduler.step()

if __name__=="__main__":
	main()


from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV

def logisticRegression(X_train, X_test, y_train, y_test):
    # Define the logistic regression model
    log_reg = LogisticRegression(penalty='l1',solver='liblinear', multi_class='ovr')

    # Define the hyperparameter grid
    param_grid = {'C': [0.01, 0.1, 1, 10, 100]}

    # Define the cross-validation object
    cv = GridSearchCV(log_reg, param_grid, cv=5)

    # Fit the cross-validation object to the training data
    cv.fit(X_train, y_train)

    return cv


__all__ = ["parse_doc", "from_string"]

import gzip
from math import atan2
import re
from . import wrapped_etree as ET
from . import svg_re
from . parse.parse_util import (
    svg_error,
    Matrix,
    clean_path_definition,
    parse_color,
    parse_color_noref,
    parse_color_stop_offset,
    dict_union,
    center_based_to_rect,
    maybe_id_ref,
    extract_local_xlink_href,
    match_children,
    parse_embedded_image_data,
    ABSOLUTE_UNIT_FACTORS
)

from . util import (
    arrow_line_end,
    deg2rad,
    rad_angle,
)

from . parse import (
    ParseState,
    get_style_dict,
    svg_length_attr,
    svg_length_attr_dumb,
)

import faint
import faint.svg.tag as tag
ns_svg = tag.ns_svg
ns_faint = tag.ns_faint

from . ifaint_util import (mul_matrix_tri,
                           to_faint_cap_str,
                           fill_style)

_DEBUG_SVG = False

def getter_default(node, default_value):
    return lambda attr : node.get(attr, default_value)

def parse_color_stops(nodes, state):
    """Parses the color stop nodes for a gradient node.
    Returns a list of offsets to colors.

    """
    stop_list = []
    for stop in [st for st in nodes if st.tag == ns_svg('stop')]:
        color_str, opacity_str = parse_color_stop_style(stop.get('style', ''))
        color_str = stop.get('stop-color', color_str)
        opacity_str = stop.get('stop-opacity', opacity_str)
        offset = parse_color_stop_offset(stop.get("offset"))

        # Fixme: Parse opacity str instead of just floating it
        faint_color = parse_color_noref(color_str, float(opacity_str), state)
        stop_list.append((offset, faint_color))
    return stop_list

def set_name(node, object_id, props):
    name = node.get('id')
    if name is not None:
        props.set_obj_name(object_id, name)

def parse_defs(node, state, id_to_etree_node=None):
    """Recursively parses an SVG defs-node."""
    if id_to_etree_node is None:
        id_to_etree_node = {}
    for child in node:
        ref_id = child.get('id')
        if ref_id is not None:
            id_to_etree_node[ref_id] = child

    for child, func in match_children(node, svg_defs_content_functions):
        item = func(child, state)

        if item is None or item.__class__ is int:
            # I guess "None" is failed parsing? Could
            # throw instead.
            # Fixme: int is probably an object id.
            # Should be able to reference those as well.
            state.add_warning("Ignored referenced item in <defs>")

        elif item == 'marker':
            # No warning on ignored markers for now, since I add them for
            # arrowhead, arrowtail.
            # Fixme: These should be indexed for referencing.
            pass
        elif item == 'calibration' or item == 'grid':
            # Fully handled by function.
            pass
        else:
            ref_id, obj = item
            state.add_ref_item(ref_id, obj)

class SvgLiteralError(faint.LoadError):
    """Exception raised for invalidly expressed SVG literals."""
    def __init__(self, literal_type, literal):
        super().__init__("Invalid %s: %s" % (literal_type, literal))
        self.literal = literal

class SvgLengthError(SvgLiteralError):
    """Exception raised for invalidly expressed SVG lengths."""
    def __init__(self, literal):
        super().__init__("length", literal)

class SvgCoordError(SvgLiteralError):
    """Exception raised for invalidly expressed SVG coordinates."""
    def __init__(self, literal):
        super().__init__("coordinate", literal)

def parse_points(points_str):
    """Parses the points specification for polyline and polygon
    elements (SVG 1.1, 9.7.1).

    """

    # Treat #-# as # -#
    points_str = points_str.replace('-', ' -')
    return [float(s) for s in re.split("[\x20\x09\x0D\x0A]+|[,]",
      points_str) if s != ""]

def parse_pattern_node(node, state, id_to_etree_node):
    """Parses a pattern element"""

    state = state.updated(node)

    node_id = node.get('id')
    if node_id is None:
        # Ignore patterns without identifiers
        return None

    for child in node:
        if child.tag == tag.image:
            encoding, data = parse_embedded_image_data(child.get(tag.xlink.href))
            if encoding == "png":
                return node_id, faint.Pattern(faint.bitmap_from_png_string(data))
    state.add_warning('Failed parsing pattern with id=%s' % node_id)
    return None

def get_linked_stops(xlink, state, id_to_etree_node):
    """Returns a referenced list of color-stops for a gradient.

    Retrieves the referenced id from the xlink, and retrieves its
    stops, if already available as a faint Gradient object in the state,
    otherwise looks up the target node in id_to_etree node, and tries to
    parse it as a gradient.

    """

    ref_id = extract_local_xlink_href(xlink)
    linked = state.ids.get(ref_id, None)

    if linked is None:
        if ref_id in id_to_etree_node:
            other = id_to_etree_node[ref_id]
            if other.tag == ns_svg('linearGradient'):
                linked_id, linked = parse_linear_gradient_node(other, state, id_to_etree_node)
            elif other.tag == ns_svg('radialGradient'):
                linked_id, linked = parse_radial_gradient_node(other, state, id_to_etree_node)
        if linked is None:
            state.add_warning("Referenced gradient not found: %s " % ref_id)

    if linked is not None:
        stops = []
        for num in range(linked.get_num_stops()):
            stops.append(linked.get_stop(num))
        return stops
    return []

def parse_faint_calibration(node, state, id_to_etree_node=None):
    points = tuple([svg_coord_attr(node.get(s,'0'), state)
              for s in ('x1','y1','x2','y2')])

    length = svg_length_attr(node.get('length'), state)
    unit = node.get('unit')
    state.props.set_calibration((points, length, unit))
    return 'calibration' # Fixme: To avoid warning. Find better method

def parse_faint_grid(node, state, id_to_etree_node=None):
    x = svg_coord_attr(node.get('x'), state)
    y = svg_coord_attr(node.get('y'), state)
    dashed = node.get('dashed') == 'True'
    enabled = node.get('enabled') == 'True'
    spacing = svg_length_attr(node.get('spacing'), state)
    state.set_grid((x, y), dashed, enabled, int(spacing))
    return 'grid' # Fixme: To avoid warning. Find better method

def parse_linear_gradient_node(node, state, id_to_etree_node=None):
    """Parses a linear gradient element"""

    state = state.updated(node)

    node_id = node.get('id')
    if node_id is None:
        # Ignore gradients without identifiers
        return None

    x1 = svg_coord_attr(node.get('x1', '0.0'), state)
    x2 = svg_coord_attr(node.get('x2', '0.0'), state)
    y1 = svg_coord_attr(node.get('y1', '0.0'), state)
    y2 = svg_coord_attr(node.get('y2', '0.0'), state)

    dx = x2 - x1
    dy = y2 - y1
    angle = atan2(dy, dx)

    stops = []
    stops = parse_color_stops(list(node), state)

    if len(stops) == 0:
        # No color stops, follow references
        # FIXME: Can there both be stops and a reference?
        xlink = node.get(tag.xlink.href)
        if xlink is not None:
            stops = get_linked_stops(xlink, state, id_to_etree_node)
        else:
            # Ignore gradients without stops
            # (Note: Need to add handling of referenced gradients)
            state.add_warning("linearGradient with id=%s has no color-stops" % node_id)
            return None
    if len(stops) == 0:
        state.add_warning("linearGradient with id=%s has no color-stops" % node_id)

        return None

    return node_id, faint.LinearGradient(angle, *stops)

def parse_radial_gradient_node(node, state, id_to_etree_node=None):
    """Parses a radial gradient node."""
    # Fixme: Review, from old
    node_id = node.get('id')
    if node_id is None:
        # Ignore gradients without identifiers
        return None

    state = state.updated(node)

    stops = []
    stops = parse_color_stops(list(node), state)

    if len(stops) == 0:
        # No color stops, follow references
        # FIXME: Can there both be stops and a reference?
        xlink = node.get(tag.xlink.href)
        if xlink is not None:
            stops = get_linked_stops(xlink, state, id_to_etree_node)
        else:
            # Ignore gradients without stops
            state.add_warning("radialGradient with id=%s has no color-stops" % node_id)
            return None
    if len(stops) == 0:
        state.add_warning("radialGradient with id=%s has no color-stops" % node_id)
        return None

    return node_id, faint.RadialGradient(stops)

def parse_color_stop_style(style):
    """Return the color and opacity from the style as strings.
    Returns None for color-string and 1.0 for opacity if not present

    """

    style_dict = get_style_dict(style)
    stop_opacity = style_dict.get('stop-opacity', '1.0')
    color_str = style_dict.get('stop-color', None)
    return color_str, stop_opacity

def parse_faint_tri_attr(node):
    """Parses a faint:tri attribute from the node"""
    tri_str = node.get(ns_faint("tri"))
    p0, p1, p2 = pairs(parse_points(tri_str))
    return faint.Tri(p0, p1, p2)

def parse_ellipse(node, state):
    """Parses an SVG <ellipse>-element."""
    state = state.updated(node)

    get = getter_default(node, '0')
    cx, cy = get('cx'), get('cy')
    cx, cy = svg_point_attr(cx, cy, state)

    rx, ry = get('rx'), get('ry')
    rx, ry = svg_point_attr(rx, ry, state)

    bounding_rect = center_based_to_rect(cx, cy, rx, ry)
    return state.props.Ellipse(bounding_rect, state.settings)

def parse_group(node, state, id_to_etree_node=None):
    """Parses an SVG <g>-element."""
    # Fixme: Use id_to_etree_node
    state = state.updated(node)
    # Parse the children of the group
    object_ids = []
    for child, func in match_children(node, included_svg_group_content_functions):
        result = func(child, state)
        if result.__class__ is not int:
            # Expected an object id from the function
            # Fixme: maybe_id_ref uses parentheses which looks weird here.
            state.add_warning("Failed parsing child %s%s of group%s" %
                              (child.tag, maybe_id_ref(child), maybe_id_ref(node)))
        else:
            object_ids.append(result)

    if len(object_ids) == 0:
        # Empty group
        return None

    return state.props.Group(object_ids)

def parse_path(node, state):
    """Parses an SVG <path>-element"""

    state = state.updated(node)
    path_def = node.get('d')
    if path_def is None:
        state.add_warning("Ignored path-element without definition "
                          "attribute%s." % maybe_id_ref(node))
        return

    path_def = clean_path_definition(path_def)
    if len(path_def) == 0:
        state.add_warning("Ignored path-element without definition "
                          "attribute%s." % maybe_id_ref(node))
        return

    try:
        path_id = state.props.Path(path_def, state.settings)
    except ValueError:
        faint.copy_text(node.get('d')) # Fixme: Remove
        state.add_warning("Failed parsing a path definition%s." %
                          maybe_id_ref(node))
        return

    return path_id

def parse_path_as_ellipse(node, state):
    """Parses an ellipse with a faint:tri attribute to an ellipse.

    Faint saves ellipses as paths for some reason (probably to
    skew/transform only the points, not the stroked edge).

    """
    state = state.updated(node)
    tri = parse_faint_tri_attr(node)
    # Fixme: Accept Tri in props.Ellipse
    ellipse_id = state.props.Ellipse((0, 0, 1, 1), state.settings)
    state.props.set_obj_tri(ellipse_id, tri)
    return ellipse_id

def parse_polygon(node, state):
    """Parses an SVG <polygon>-element."""
    state = state.updated(node)
    points = parse_points(node.get('points', ''))
    if len(points) % 2 != 0:
        state.add_warning("Odd number of coordinates for polygon%s." %
                          maybe_id_ref(node))
        # SVG 1.1 F2 "Error processing" says
        # Render polyline and polygons with invalid points up to the
        # erroneous point.
        points = points[:-1]

    return state.props.Polygon(points, state.settings)

def parse_polyline(node, state):
    """Parses an SVG <polyline>-element."""
    state = state.updated(node)

    points = parse_points(node.get('points'))

    if state.settings.arrow == 'front':
        x0, y0, x1, y1 = points[-4:]
        angle = rad_angle(x0,y0,x1,y1)
        x, y = arrow_line_end(x1, y1, angle, state.settings.linewidth)
        points[-2] = x
        points[-1] = y

    if len(points) % 2 != 0:
        state.add_warning("Odd number of coordinates for polyline%s." %
                          maybe_id_ref(node))
        # SVG 1.1 F2 "Error processing" says
        # Render polyline and polygons with invalid points up to the
        # erroneous point.
        points = points[:-1]

    return state.props.Line(points, state.settings)

def parse_faint_text_size_attrs(node, state):
    """Parses the faint-added width, height attributes"""

    # Get name-spaced or un-namespaced width, height
    # (added by Faint regardless)
    width_str = (node.get(ns_faint('width')) or
                  node.get('width'))
    height_str = (node.get(ns_faint('height')) or
                  node.get('height'))
    if width_str is not None:
        w = svg_length_attr(width_str, state)
    else:
        w = 200 # Fixme
    if height_str is not None:
        h = svg_length_attr(height_str, state)
    else:
        h = 200
    return not (width_str is None or height_str is None), w, h

def svg_baseline_to_faint(text_id, state):
    """Convert the position of the text object in props with text_id.

    SVG anchors at baseline, Faint at top of text.
    """
    dy = state.props.get_obj_text_height(text_id)
    tri = state.props.get_obj_tri(text_id)
    tri.translate(0, -dy)
    state.props.set_obj_tri(text_id, tri)

def parse_text(node, state):
    """Parses an SVG <text>-element."""
    state = state.updated(node)

    # Fixme: Should support <coordinate-list>
    get = getter_default(node, '0')
    x, y = svg_point_attr(get('x'), get('y'), state)

    boxed, w, h = parse_faint_text_size_attrs(node, state)
    # Fixme: Parse dx, dy lists etc.

    settings = faint.Settings()
    settings.update_all(state.settings)

    # Fixme: These assignments will overwrite inherited settings
    #        even if node doesn't have the retrieved setting
    default_bound = '1' if boxed else '0'
    settings.bounded = node.get(ns_faint('bounded'), default_bound) == '1'

    settings.valign = node.get(ns_faint('valign'), 'top')

    # TODO: If no unit specified size is processed as a height in the user
    #       coordinate system.
    #       Otherwise converts the length into user coordinate system

    fontsize_str = node.get('font-size')
    if fontsize_str is not None:
        # Fixme: Handle 'inherit' generically somehow
        if fontsize_str != 'inherit':
            settings.fontsize = svg_length_attr(fontsize_str, state)

    style = get_style_dict(node.get('style', ''))
    if 'fill' in style:
        settings.fg = parse_color(style['fill'], "1.0", state)

    anchor = node.get('text-anchor', 'start')
    if anchor == 'start':
        settings.halign = 'left'
    elif anchor == 'middle':
        settings.halign = 'center'
        x -= w / 2 # fixme
    elif anchor == 'end':
        settings.halign = 'right'

    text_str = None
    settings.parsing = node.get(ns_faint("parsing")) == '1'
    if settings.parsing:
        raw_node = node.find(ns_faint('raw'))
        if raw_node is not None:
            text_str = raw_node.text

    if text_str is None:
        text_str = ""
        if node.text:
            text_str += node.text

        for child in node:
            if child.tag == ns_svg('tspan'):
                # Fixme: Parse settings from the tspans
                if len(child.text) > 0:
                    text_str += child.text

                if child.get(ns_faint('hardbreak'), '0') == '1':
                    text_str += "\n"

    text_id = state.props.Text((x, y, w, h), text_str, settings)
    svg_baseline_to_faint(text_id, state)
    return text_id

def is_faint_type(node, expected):
    """True if the node has a faint:type attribute with value matching
    expected. Allows mapping SVG shapes etc. to specific Faint object
    types.

    """
    return node.get(ns_faint('type')) == expected

def pairs(iterable):
    """Return a list which pairs up the neighbouring items in tuples.
    Will lose the last item if uneven item count.
    """
    i = iter(iterable)
    return list(zip(i, i))

def parse_circle(node, state):
    """Parses an SVG <circle>-element."""
    state = state.updated(node)
    cx = svg_coord_attr(node.get('cx', '0'), state)
    cy = svg_coord_attr(node.get('cy', '0'), state)
    r = svg_length_attr(node.get('r', '0'), state)
    rect = center_based_to_rect(cx, cy, r, r)
    return state.props.Ellipse(rect, state.settings)

def parse_polygon_as_rect(node, state):
    """Skewed Faint-rectangles are saved as polygons (marked with
    faint:type="rect").

    """
    state = state.updated(node)

    name = node.get('id')

    # Fixme: Pass Tri to props.rect instead of defaulting like this
    rect_id = state.props.Rect((0, 0, 1, 1), state.settings, name)

    p0, p1, p3, p2 = pairs(parse_points(node.get('points')))
    del p3
    state.props.set_obj_tri(rect_id, faint.Tri(p0, p1, p2))
    return rect_id

def parse_image_as_background(node, state):
    """Parses the embedded content of image node with a faint:background
    attribute and sets it as the background.

    """
    image_str = node.get(tag.xlink.href)
    image_type, data = parse_embedded_image_data(image_str)
    assert image_type == 'png'
    state.props.set_background(faint.bitmap_from_png_string(data))

def parse_image(node, state):
    """Parses an SVG <image>-element."""
    assert node.get(ns_faint('background')) is None
    state = state.updated(node)

    x = svg_length_attr(node.get('x', '0.0'), state)
    y = svg_length_attr(node.get('y', '0.0'), state)
    w = svg_length_attr(node.get('width', '0.0'), state)
    h = svg_length_attr(node.get('height', '0.0'), state)

    image_str = node.get(tag.xlink.href)
    if image_str is None:
        state.add_warning("Ignored image element with no data.")
        return

    image_type, image_data = parse_embedded_image_data(image_str)
    bitmap_func = IMAGE_TYPE_TO_BITMAP.get(image_type)
    if bitmap_func is None:
        state.add_warning('Ignored image element with unsupported type: %s' %
            image_type)
        return

    bmp = bitmap_func(image_data)
    return state.props.Raster((x, y, w, h), bmp, state.settings)

def parse_image_custom(node, state):
    """Parses an SVG <image> element into a faint raster background if
    marked-up as such, otherwise as a Faint raster object.

    """
    if node.get(ns_faint('background')) is not None:
        return parse_image_as_background(node, state)
    else:
        return object_common(parse_image)(node, state)

def parse_line(node, state):
    """Parses an SVG <line> element."""
    state = state.updated(node)

    points = [svg_coord_attr(node.get(s, '0'), state)
           for s in ('x1', 'y1', 'x2', 'y2')]

    if state.settings.arrow == 'front':
        x0, y0, x1, y1 = points
        angle = rad_angle(x0,y0,x1,y1)
        x, y = arrow_line_end(x1, y1, angle, state.settings.linewidth)
        points[-2] = x
        points[-1] = y

    return state.props.Line(points, state.settings)

def parse_path_custom(node, state):
    """Parses an SVG <path> element as a Faint Ellipse if marked-up as
    such, otherwise as a Faint Path.

    """
    return (parse_path_as_ellipse(node, state) if is_faint_type(node, 'ellipse')
            else parse_path(node, state))

def parse_polygon_custom(node, state):
    """Parses an SVG <polygon> element as a Faint Rectangle if marked-up
    as such, otherwise as a Faint Polygon.

    """
    return (parse_polygon_as_rect(node, state) if is_faint_type(node, 'rect')
            else parse_polygon(node, state))

def parse_rect_as_background(node, state):
    """Parses a marked-up rect-node to a background color.

    """
    color = parse_color(node.get('fill'), '1.0', state)
    state.props.set_background((color, state.props.get_size()))

def parse_rect(node, state):
    """Parses an SVG <rect> element."""
    state = state.updated(node)

    get = getter_default(node, '0')

    x, y = svg_point_attr(get('x'), get('y'), state)
    w, h = svg_size_attr(get('width'), get('height'), state)

    state.settings.rx = svg_length_attr(get('rx'), state)
    state.settings.ry = svg_length_attr(get('ry'), state)

    r = state.props.Rect((x, y, w, h), state.settings)
    return r

def parse_rect_custom(node, state):
    """Parses an SVG <rect> element as the Canvas background color if
    marked-up as such, otherwise as a regular Rectangle.

    """
    if node.get(ns_faint('background')) is not None:
        return parse_rect_as_background(node, state)
    else:
        return object_common(parse_rect)(node, state)

def parse_switch(node, state):
    """Parses a switch node"""
    for child in node:
        if child.tag not in svg_switch_element_content_functions:
            continue

        requiredExtensions = child.get('requiredExtensions')
        if requiredExtensions is not None and requiredExtensions != "":
            # Faint supports no extensions.
            continue

        requiredFeatures = child.get('requiredFeatures')
        if requiredFeatures is not None:
            rfs = requiredFeatures.split(" ")
            unsupported = [rf.strip() for rf in rfs if
                           rf not in supported_svg_features]
            if len(unsupported) != 0:
                # At least one required feature was unsupported.
                continue

        systemLanguage = child.get('systemLanguage', state.system_language)
        if systemLanguage == state.system_language:
            return svg_switch_element_content_functions[child.tag](child, state)

    state.add_warning("No supported child node of SVG switch-element")

def _not_implemented(node, state, *args): #pylint:disable=unused-argument
    if _DEBUG_SVG:
        state.add_warning("Support not implemented: %s" % node.tag)

def _not_implemented_marker(node, state, *args): #pylint:disable=unused-argument
    if _DEBUG_SVG:
        state.add_warning("Support not implemented: %s" % node.tag)
    return 'marker'

# Dictionary of functions for converting embedded image-element data
# to a Bitmap. The data must be decoded from e.g. base64 first.
IMAGE_TYPE_TO_BITMAP = {
    "jpeg": faint.bitmap_from_jpg_string,
    "png": faint.bitmap_from_png_string,
}

# pylint:disable=invalid-name

svg_animation_element_functions = {
    ns_svg('animate') : _not_implemented,
    ns_svg('animateColor') : _not_implemented,
    ns_svg('animateMotion') : _not_implemented,
    ns_svg('animateTransform'): _not_implemented,
    ns_svg('set'): _not_implemented
}

svg_descriptive_element_functions = {
    ns_svg('desc'): _not_implemented,
    ns_svg('metadata'): _not_implemented,
    ns_svg('title'): _not_implemented
}

svg_gradient_element_functions = {
    # SVG 1.1
    ns_svg('linearGradient') : parse_linear_gradient_node,
    ns_svg('radialGradient') : parse_radial_gradient_node
}

def object_common(func):
    """Wraps an object-parse function, and adds common functionality,
    instead of scattering it in the individual parse functions.

    """
    def update_object(node, state):
        obj_id = func(node, state)
        if obj_id is not None:
            state.transform_object(obj_id)
            set_name(node, obj_id, state.props)
        return obj_id

    return update_object

svg_shape_element_functions = {
    tag.circle : object_common(parse_circle),
    tag.ellipse : object_common(parse_ellipse),
    tag.line : object_common(parse_line),
    tag.path : object_common(parse_path_custom),
    tag.polygon: object_common(parse_polygon_custom),
    tag.polyline : object_common(parse_polyline),
    tag.rect : parse_rect_custom,
}

svg_structural_element_functions = {
    tag.defs: parse_defs,
    tag.g: object_common(parse_group),
    tag.svg: _not_implemented, # nested SVG
    tag.symbol: _not_implemented,
    tag.use: _not_implemented
}

svg_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    svg_shape_element_functions,
    svg_structural_element_functions,
    svg_gradient_element_functions,
    {
        tag.a: _not_implemented,
        ns_svg('altGlyphDef'): _not_implemented,
        ns_svg('clipPath'): _not_implemented,
        ns_svg('color-profile'): _not_implemented,
        ns_svg('cursor'): _not_implemented,
        ns_svg('filter'): _not_implemented,
        ns_svg('font'): _not_implemented,
        ns_svg('font-face'): _not_implemented,
        tag.foreignObject: _not_implemented,
        tag.image : parse_image_custom,
        ns_svg('marker'): _not_implemented_marker,
        ns_svg('mask'): _not_implemented,
        ns_svg('pattern'): _not_implemented,
        ns_svg('script'): _not_implemented,
        ns_svg('style'): _not_implemented,
        tag.switch: parse_switch,
        tag.text: object_common(parse_text),
        tag.view: _not_implemented
    }
)

if _DEBUG_SVG:
    ignored_svg_group_content = []
else:
    ignored_svg_group_content = [
        # Exclude certain elements unsupported by Faint in group nodes,
        # to remove warnings about failing parsing a child for hopefully
        # less important children
        ns_svg('title')
    ]

svg_group_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    svg_shape_element_functions,
    svg_structural_element_functions,
    svg_gradient_element_functions,
    {
        tag.a: _not_implemented,
        ns_svg('altGlyphDef'): _not_implemented,
        ns_svg('clipPath'): _not_implemented,
        ns_svg('color-profile') : _not_implemented,
        ns_svg('cursor'): _not_implemented,
        ns_svg('filter'): _not_implemented,
        ns_svg('font'): _not_implemented,
        ns_svg('font-face'): _not_implemented,
        tag.foreignObject: _not_implemented,

        # Fixme: Shouldn't image in group go directly for the
        # non-background parsing?
        tag.image: parse_image_custom,

        ns_svg('marker'): _not_implemented_marker,
        ns_svg('mask'): _not_implemented,
        ns_svg('pattern'): _not_implemented,
        ns_svg('script'): _not_implemented,
        ns_svg('style'): _not_implemented,
        tag.switch: parse_switch,
        tag.text: object_common(parse_text),
        tag.view: _not_implemented,
    })

included_svg_group_content_functions = {key: svg_group_content_functions[key]
                                        for key in svg_group_content_functions
                                        if key not in ignored_svg_group_content}

svg_switch_element_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    svg_shape_element_functions,
    {
        tag.a: _not_implemented,
        tag.foreignObject: _not_implemented,
        tag.g: object_common(parse_group),
        tag.image: object_common(parse_image),
        tag.svg: _not_implemented, # Nested SVG
        tag.switch: parse_switch,
        tag.svg: object_common(parse_text),
        tag.use: _not_implemented,
    })

svg_descriptive_element_functions = {
    ns_svg('desc'): _not_implemented,
    ns_svg('metadata'): _not_implemented,
    tag.title: _not_implemented,
}

svg_defs_content_functions = dict_union(
    svg_animation_element_functions,
    svg_descriptive_element_functions,
    # todo: shape
    # todo: structural
    svg_structural_element_functions,
    svg_gradient_element_functions,
    {
        tag.a: _not_implemented,
        ns_svg('altGlyphDef'): _not_implemented,
        ns_svg('clipPath'): _not_implemented,
        ns_svg('colorProfile'): _not_implemented,
        ns_svg('cursor'): _not_implemented,
        ns_svg('filter'): _not_implemented,
        ns_svg('font'): _not_implemented,
        ns_svg('font-face'): _not_implemented,
        tag.foreignObject: _not_implemented,
        tag.image: _not_implemented,
        ns_svg('marker'): _not_implemented_marker,
        ns_svg('mask'): _not_implemented,
        ns_svg('pattern'): parse_pattern_node,
        ns_svg('script'): _not_implemented,
        ns_svg('style'): _not_implemented,
        ns_svg('switch'): parse_switch,
        tag.text: _not_implemented,
        tag.view: _not_implemented,
        tag.faint.calibration: parse_faint_calibration,
        tag.faint.grid: parse_faint_grid,
    })

supported_svg_features = [

    # Not sure, group
    "http://www.w3.org/TR/SVG11/feature#SVG",

    # Not sure
    "http://www.w3.org/TR/SVG11/feature#SVGDOM",

    # Not totally.
    "http://www.w3.org/TR/SVG11/feature#SVG-static",

    # Probably untrue
    "http://www.w3.org/TR/SVG11/feature#SVGDOM-static",

    # No animation support at all
    #"http://www.w3.org/TR/SVG11/feature#SVG-animation",
    #"http://www.w3.org/TR/SVG11/feature#SVGDOM-animation",
    #"http://www.w3.org/TR/SVG11/feature#SVG-dynamic",
    #"http://www.w3.org/TR/SVG11/feature#SVGDOM-dynamic",

    # Support for attributes:
    # id, xml:base, xml:space attributes
    #"http://www.w3.org/TR/SVG11/feature#CoreAttribute",

    # Support for elements:
    # svg, g, defs, desc, title, metadata, symbol, use
    "http://www.w3.org/TR/SVG11/feature#Structure",

    # Support for elements:
    # svg, g, defs, desc, title, metadata, use
    "http://www.w3.org/TR/SVG11/feature#BasicStructure",

    # Support for enable-background property
    #"http://www.w3.org/TR/SVG11/feature#ContainerAttribute",

    # Support for the switch element, and the attributes:
    # requiredFeatures, requiredExtensions, systemLanguage
    "http://www.w3.org/TR/SVG11/feature#ConditionalProcessing",

    # Support for the image element
    "http://www.w3.org/TR/SVG11/feature#Image",

    # Support for the style element
    # (Not true)
    "http://www.w3.org/TR/SVG11/feature#Style",

    # Support for clip and overflow properties of viewport
    #"http://www.w3.org/TR/SVG11/feature#ViewportAttribute",

    # Support for the shape elements:
    # rect, circle, line, polyline, polygon, ellipse, path.
    "http://www.w3.org/TR/SVG11/feature#Shape",

    # Support for text, tspan, tref, textPath, altGlyph, altGlyphDef,
    # altGlyphItem, glyphRef.
    "http://www.w3.org/TR/SVG11/feature#Text",

    # Support for the text element.
    "http://www.w3.org/TR/SVG11/feature#BasicText",

    # Support for the properties:
    # color, fill, fill-rule, stroke, stroke-dasharray,
    # stroke-dashoffset, stroke-linecap, stroke-linejoin,
    # stroke-miterlimit, stroke-width, color-interpolation,
    # color-rendering.
    "http://www.w3.org/TR/SVG11/feature#PaintAttribute",

    # Support for the properties: color, fill, fill-rule, stroke,
    # stroke-dasharray, stroke-dashoffset, stroke-linecap,
    # stroke-linejoin, stroke-miterlimit, stroke-width,
    # color-rendering.
    "http://www.w3.org/TR/SVG11/feature#BasicPaintAttribute",

    # Support for opacity, stroke-opacity, fill-opacity
    # Fixme: 'opacity'? Check that out.
    "http://www.w3.org/TR/SVG11/feature#OpacityAttribute",

    # Support for the properties display, image-rendering,
    # pointer-events, shape-rendering, text-rendering visibility
    "http://www.w3.org/TR/SVG11/feature#GraphicsAttribute",

    # Support for display and visibility properties
    "http://www.w3.org/TR/SVG11/feature#BasicGraphicsAttribute",

    # Support for the marker element
    #"http://www.w3.org/TR/SVG11/feature#Marker",

    # Support for the color-profile element
    #"http://www.w3.org/TR/SVG11/feature#ColorProfile",

    # Support for elements linearGradient, radialGradient and stop
    "http://www.w3.org/TR/SVG11/feature#Gradient",

    # Support for the pattern element
    #"http://www.w3.org/TR/SVG11/feature#Pattern",

    # Support for the clipPath element and clip-path, clip-rule
    # properties
    #"http://www.w3.org/TR/SVG11/feature#Clip",

    # Support for the clipPath element and clip-path property
    #"http://www.w3.org/TR/SVG11/feature#BasicClip",

    # Support for the mask element
    #"http://www.w3.org/TR/SVG11/feature#Mask",

    # Support for the elements filter, feBlend, feColorMatrix,
    # feComponentTransfer, feComposite, feConvolveMatrix,
    # feDiffuseLighting, feDisplacementMap, feFlood, feGaussianBlur,
    # feImage, feMerge, feMergeNode, feMorphology, feOffset,
    # feSpecularLighting, feTile, feDistantLight, fePointLight,
    # feSpotLight, feFuncR, feFuncG, feFuncB, feFuncA.
    #"http://www.w3.org/TR/SVG11/feature#Filter",

    # Support for the elements filter, feBlend, feColorMatrix,
    # feComponentTransfer, feComposite, feFlood, feGaussianBlur,
    # feImage, feMerge, feMergeNode, feOffset, feTile, feFuncR,
    # feFuncG, feFuncB, feFuncA.
    #"http://www.w3.org/TR/SVG11/feature#BasicFilter",

    # Support for attributes onunload, onabort, onerror, onresize,
    # onscroll and onzoom.
    #"http://www.w3.org/TR/SVG11/feature#DocumentEventsAttribute",

    # Support for the attributes onfocusin, onfocusout, onactivate, onclick,
    # onmousedown, onmouseup, onmouseover, onmousemove, onmouseout,
    # onload.
    #"http://www.w3.org/TR/SVG11/feature#GraphicalEventsAttribute",

    # Support for the attributes onbegin, onend, onrepeat, onload
    #"http://www.w3.org/TR/SVG11/feature#AnimationEventsAttribute",

    # Support for the cursor element
    #"http://www.w3.org/TR/SVG11/feature#Cursor",

    # Support for the a element.
    #"http://www.w3.org/TR/SVG11/feature#Hyperlinking",

    # Support for the xlink:type, xlink:href, xlink:role,
    # xlink:arcrole, xlink:title, xlink:show and xlink:actuate
    # elements.
    # (Fixme: Only xlink:href in rare cases)
    "http://www.w3.org/TR/SVG11/feature#XlinkAttribute",

    # Support for the externalResourcesRequired attribute
    #"http://www.w3.org/TR/SVG11/feature#ExternalResourcesRequired",

    # Support for the view element.
    #"http://www.w3.org/TR/SVG11/feature#View",

    # Support for the script element.
    #"http://www.w3.org/TR/SVG11/feature#Script",

    # Support for the animate, set, animateMotion, animateTransform,
    # animateColor and mpath elements.
    #"http://www.w3.org/TR/SVG11/feature#Animation",

    # Support for the font, font-face, glyph, missing-glyph, hkern,
    # vkern, font-face-src, font-face-uri, font-face-format,
    # font-face-name elements.
    # (Fixme: Very limited support)
    "http://www.w3.org/TR/SVG11/feature#Font",

    # Support for the elements font, font-face, glyph, missing-glyph,
    # hkern, font-face-src, font-face-name.
    "http://www.w3.org/TR/SVG11/feature#BasicFont",

    # The foreignObject element
    #"http://www.w3.org/TR/SVG11/feature#Extensibility",
]

# pylint:enable=invalid-name

def svg_size_attr(w_str, h_str, state):
    """Parses a pair of coordinates, scaling percentages."""

    return (svg_length_attr_dumb(w_str, state.props, state.viewPort[2]),
            svg_length_attr_dumb(h_str, state.props, state.viewPort[3]))

def svg_point_attr(x_str, y_str, state):
    """Parses a pair of coordinates, scaling percentages."""

    return (svg_length_attr_dumb(x_str, state.props, state.viewPort[2]),
            svg_length_attr_dumb(y_str, state.props, state.viewPort[3]))

def svg_coord_attr(coord_str, state):
    """Parses an svg coordinate attribute."""
    # Fixme: Duplicates length_attr
    m = re.match(svg_re.coordinate, coord_str)
    if m is None:
        raise SvgCoordError(coord_str)
    value, unit = m.groups()
    if unit == '%':
        # Fixme: How should this work?
        if float(value) == 0:
            return 0
        else:
            return (state.viewPort[2] * 100) / float(value)
    elif unit in ABSOLUTE_UNIT_FACTORS:
        return float(value) * ABSOLUTE_UNIT_FACTORS[unit]
    elif unit in ('em', 'ex'):
        state.add_warning("Unsupported unit: %s" % unit)
        return float(value)
    else:
        state.add_warning("Invalid unit: %s" % unit)
        return float(value)

def check_svg_size(size):
    """Raises exception if invalid size for SVG node."""
    if size[0] <= 0:
        raise faint.LoadError("SVG element has negative width")
    elif size[1] <= 0:
        raise faint.LoadError("SVG element has negative height")
    return size

# Fixme: Use image width, height for default viewbox
def get_viewbox(node):
    """Returns the viewbox from the node as a sequence of four strings,
    x, y, w, h."""

    # Available for: svg, symbol (via use), image, foreignObject
    viewBox = node.get('viewBox') # pylint:disable=invalid-name
    if viewBox is None:
        return None

    # Fixme: Handle preserveAspectRatio
    # Fixme: Can also use commas
    x, y, w, h = [float(v) for v in viewBox.split(' ')]
    return x, y, w, h

def establish_viewPort(svg_node, image_props, view_box):
    # Use the viewBox width and height if width, height not specified
    if view_box is not None:
        default_w = str(view_box[2])
        default_h = str(view_box[3])
    else:
        default_w = "640.0"
        default_h = "480.0"

    svg_w = svg_node.get('width', None)
    svg_h = svg_node.get('height', None)

    if svg_w is None:
        svg_w = default_w
    if svg_h is None:
        svg_h = default_h

    # Fixme: I guess viewbox should be converted, not float:ed
    w2 = svg_length_attr_dumb(svg_w, image_props, full_span=float(default_w))
    h2 = svg_length_attr_dumb(svg_h, image_props, full_span=float(default_h))
    w2, h2 = check_svg_size((w2, h2))

    return [0, 0, w2, h2]

def parse_svg_root_node(svg_node, image_props, system_language):
    """Parses the SVG-root node and recurses into child nodes. Creates the
    frame from the SVG root node properties.

    """

    # Fixme: Check if SVG element can have transform (Probably)
    viewBox = get_viewbox(svg_node)
    viewPort = establish_viewPort(svg_node, image_props, viewBox)
    if viewBox is None:
        viewBox = viewPort

    frame_props = image_props.add_frame((viewPort[2], viewPort[3]))

    # Fixme: What of translation?
    ctm = Matrix.scale(viewPort[2] / viewBox[2], viewPort[3] / viewBox[3])

    state = ParseState(image_props,
                       frame_props,
                       ctm=ctm,
                       viewPort=viewPort,
                       viewBox=viewBox,
                       system_language=system_language)

    for child in svg_node:
        if child.tag in svg_content_functions:
            # Fixme: I guess there could be a transform already?
            svg_content_functions[child.tag](child, state)

def parse_svgz_file(path, image_props, system_language):
    with gzip.open(path, 'rb') as f:
        file_content = f.read()
    return from_string(file_content, image_props)

def _parse_svg_file(path, image_props, system_language):
    ET.register_namespace("svg", "{http://www.w3.org/2000/svg}")
    tree = ET.parse(path)
    root = tree.getroot()
    if root.tag == ns_svg('svg'):
        parse_svg_root_node(root, image_props, system_language)
    else:
        raise faint.LoadError("Root element was not <svg>.")

def parse_doc(path, image_props, system_language="en"):
    """Parses the SVG document at the given file path, using the
    image_props to build an image.

    """

    try:
        if path.endswith(".svgz"):
            parse_svgz_file(path, image_props, system_language)
        else:
            _parse_svg_file(path, image_props, system_language)

    except svg_error as e:
        raise faint.LoadError(str(e))

def from_string(xml_string, image_props, system_language="en"):
    """Parses the SVG document in the given string, using the image_props
    to build an image.

    """

    try:
        ET.register_namespace("svg", "{http://www.w3.org/2000/svg}")
        root = ET.fromstring(xml_string)
        if root.tag == ns_svg("svg"):
            parse_svg_root_node(root, image_props, system_language)
        else:
            raise faint.LoadError("Root element was not <svg>.")
    except svg_error as e:
        raise faint.LoadError(str(e))


from flask import Flask, render_template, request, redirect
from flask_sqlalchemy import SQLAlchemy
import json
import requests
from datetime import datetime
from bs4 import BeautifulSoup
import re
import urllib.parse
from dateutil.parser import parse as dateparse
from threading import Thread
from time import sleep
import configparser

c = configparser.ConfigParser()
c.read('config.cfg')
c = c['MAIN']
db_path = 'mysql+pymysql://{}:{}@{}:{}/{}'.format(c['MYSQL_USER'],
    c['MYSQL_PASS'], c['MYSQL_HOST'], c['MYSQL_PORT'], c['MYSQL_DB'])
server_host = c['SERVER_HOST']
server_port = c['SERVER_PORT']
nyaa_host = c['NYAA_HOST']

# db_path = 'mysql+pymysql://root:mysql@localhost:32775/db1'
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = db_path
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

def fetch_nyaa(search, user):
    search = urllib.parse.quote(search)
    user = urllib.parse.quote(user)
    ss = 'https://{}/user/{}?f=0&c=0_0&q={}'
    req = ss.format(nyaa_host, user, search)
    data = requests.get(req).text
    s = BeautifulSoup(data)
    table = s.find('table', class_='torrent-list')
    rows = table.tbody.findAll('tr')
    episodes = []
    for row in rows:
        cells = row.findAll('td')
        try:
            link = cells[1].findAll('a')
            lstr = ''
            for l in link:
                if 'comment' not in l.get('title'):
                    lstr = l.string
            magnet = cells[2].find('a', href=re.compile('magnet:?'))
            date = dateparse(cells[4].string)
            episodes.append(
                {
                    'name': lstr,
                    'link': magnet.get('href'),
                    'date': date
                }
            )
        except:
            pass
    return episodes

site_methods = {
    'nyaa': fetch_nyaa
}

def fetch_episodes(site, search, user=None):
    m = site_methods.get(site)
    if m is None:
        return None
    return m(search, user)

class show(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(80), unique=True, nullable=False)
    site = db.Column(db.String(80), nullable=False)
    search = db.Column(db.String(80), nullable=False)
    user = db.Column(db.String(80), nullable=False)

    @staticmethod
    def add_show():
        return None

    def get_episodes(self):
        current_eps = [e.name for e in self.episodes]
        eps = fetch_episodes(self.site, self.search, self.user)

        for ep in eps:
            if(ep['name'] not in current_eps):
                e = episode(name=ep['name'], link=ep['link'], date=ep['date'], watched=False)

                self.episodes.append(e)
        db.session.add(self)
        db.session.commit()

    def get_next_episode(self):
        eps = [[e.name, e.link, e.date] for e in self.episodes if e.watched == False]
        mi = ['No New Episode', '' , datetime.now()]
        for e in eps:
            if e[2] < mi[2]:
                mi = e
        return mi

    def delete(self):
        for ep in self.episodes:
            db.session.delete(ep)
        db.session.commit()
        db.session.delete(self)
        db.session.commit()

    def __repr__(self):
        return self.name + self.site + self.search + self.user + str([e.name for e in self.episodes])

class episode(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(100), unique=True, nullable=False)
    link = db.Column(db.String(500), nullable=False)
    date = db.Column(db.DateTime)
    show_id = db.Column(db.Integer, db.ForeignKey('show.id'),
        nullable=False)
    show = db.relationship('show', backref=db.backref('episodes', lazy=True))
    watched = db.Column(db.Boolean)

    def __repr__(self):
        return '{} {} {} {}'.format(self.name, self.link, str(self.date), str(self.watched))

@app.route('/')
def m():
    s = db.session.query(show).all()
    shows = [[sh.name, sh.id, sh.get_next_episode()[0]] for sh in s]
    return render_template('main.html', shows=shows)

@app.route('/add_show', methods=['GET', 'POST'])
def add_how():
    message = False
    message_string = ''
    if request.method == "POST":
        message = True
        try:
            s = show(name=request.form["sname"], site=request.form["site"], search=request.form["sstring"], user=request.form["uploader"])
            s.get_episodes()
            message_string = 'Show Added'
        except:
            message_string = 'Something Went wrong'

    return render_template('add_show.html', message=message, message_string=message_string)

@app.route('/show/<show_id>')
def show_info(show_id):
    s = db.session.query(show).filter(show.id == show_id).one_or_none()
    if not s:
        return 'show not found'
    eps = [[e.name, e.date, e.id, e.watched] for e in s.episodes]
    eps = sorted(eps, key=lambda ep: ep[1])

    return render_template('show.html', name=s.name, sid=s.id, eps=eps[::-1])

@app.route('/download_next/<show_id>')
def download(show_id):
    s = db.session.query(show).filter(show.id == show_id).one_or_none()
    if not s:
        return 'show not found'
    next_ep = s.get_next_episode()
    ep = db.session.query(episode).filter(episode.name == next_ep[0]).one_or_none()
    if ep:
        ep.watched = True
        db.session.add(ep)
        db.session.commit()
    return redirect(next_ep[1], code=302)

@app.route('/download_ep/<ep_id>')
def download_ep(ep_id):
    ep = db.session.query(episode).filter(episode.id == ep_id).one_or_none()
    if ep:
        return redirect(ep.link, code=302)
    return ''

@app.route('/watch_ep/<ep_id>')
def watch_ep(ep_id):
    ep = db.session.query(episode).filter(episode.id == ep_id).one_or_none()
    if ep:
        ep.watched = not ep.watched
        db.session.add(ep)
        db.session.commit()
    return redirect('/show/' + str(ep.show_id), code=302)

@app.route('/delete_show/<show_id>')
def delete_show(show_id):
    s = db.session.query(show).filter(show.id == show_id).one_or_none()
    if s:
        s.delete()
    return redirect('/', code=302)

@app.route('/update_shows')
def update_shows():
    s = db.session.query(show).all()
    for sh in s:
        try:
            sh.get_episodes()
        except:
            pass
    return 'success'

def show_updater():
    while True:
        sleep(1200)
        requests.get('http://{}:{}/update_shows'.format(server_host, server_port))

if __name__ == "__main__":
    db.create_all()
    thread = Thread(target=show_updater)
    thread.start()
    app.run(host=server_host, port=int(server_port))


# -*- coding: utf-8 -*-
#------------------------------------------------------------
# pelisalacarta - XBMC Plugin
# Canal para piratestreaming
# http://blog.tvalacarta.info/plugin-xbmc/pelisalacarta/
#------------------------------------------------------------
import urlparse,urllib2,urllib,re
import os, sys

from core import logger
from core import config
from core import scrapertools
from core.item import Item
from servers import servertools

__channel__ = "liberoita"
__category__ = "F"
__type__ = "generic"
__title__ = "Liberoita (IT)"
__language__ = "IT"

DEBUG = config.get_setting("debug")

def isGeneric():
    return True

def mainlist(item):
    logger.info("pelisalacarta.liberoita mainlist")
    itemlist = []
    itemlist.append( Item(channel=__channel__, title="[COLOR azure]Novita'[/COLOR]", action="peliculas", url="http://liberoita.org/", thumbnail="http://dc584.4shared.com/img/XImgcB94/s7/13feaf0b538/saquinho_de_pipoca_01"))
    itemlist.append( Item(channel=__channel__, title="[COLOR azure]Categorie[/COLOR]", action="categorias", url="http://liberoita.org/", thumbnail="http://xbmc-repo-ackbarr.googlecode.com/svn/trunk/dev/skin.cirrus%20extended%20v2/extras/moviegenres/All%20Movies%20by%20Genre.png"))
    itemlist.append( Item(channel=__channel__, title="[COLOR yellow]Cerca...[/COLOR]", action="search", thumbnail="http://dc467.4shared.com/img/fEbJqOum/s7/13feaf0c8c0/Search"))

    return itemlist

def categorias(item):
    logger.info("pelisalacarta.liberoita categorias")
    itemlist = []

    data = scrapertools.cache_page(item.url)
    logger.info(data)

    # Narrow search by selecting only the combo
    bloque = scrapertools.get_match(data,'
(.*?)
')

    # The categories are the options for the combo
    patron = '
]+>]+>(.*?)\s*
'
    matches = re.compile(patron,re.DOTALL).findall(bloque)
    scrapertools.printMatches(matches)

    for url,titulo in matches:
        scrapedtitle = titulo
        scrapedurl = urlparse.urljoin(item.url,url)
        scrapedthumbnail = ""
        scrapedplot = ""
        if (DEBUG): logger.info("title=["+scrapedtitle+"], url=["+scrapedurl+"], thumbnail=["+scrapedthumbnail+"]")
        itemlist.append( Item(channel=__channel__, action="peliculas" , title="[COLOR azure]"+scrapedtitle+"[/COLOR]" , url=scrapedurl, thumbnail=scrapedthumbnail, plot=scrapedplot))

    return itemlist

def search(item,texto):
    logger.info("[liberoita.py] "+item.url+" search "+texto)
    item.url = "http://liberoita.org/?s="+texto+"&x=0&y=0"
    try:
        return peliculas(item)
    # Se captura la excepción, para no interrumpir al buscador global si un canal falla
    except:
        import sys
        for line in sys.exc_info():
            logger.error( "%s" % line )
        return []

def peliculas(item):
    logger.info("pelisalacarta.liberoita peliculas")
    itemlist = []

    # Descarga la pagina
    data = scrapertools.cache_page(item.url)

    # Extrae las entradas (carpetas)
    patron  = '
\s*'
    patron  += ''
    matches = re.compile(patron,re.DOTALL).findall(data)
    scrapertools.printMatches(matches)

    for scrapedurl,scrapedthumbnail,scrapedtitle in matches:
        scrapedtitle = scrapertools.decodeHtmlentities(scrapedtitle)
        response = urllib2.urlopen(scrapedurl)
        html = response.read()
        start = html.find("
")
        end = html.find("
", start)
        scrapedplot = html[start:end]
        scrapedplot = re.sub(r'<[^>]*>', '', scrapedplot)
        #scrapedplot = ""
        if (DEBUG): logger.info("title=["+scrapedtitle+"], url=["+scrapedurl+"], thumbnail=["+scrapedthumbnail+"]")
        itemlist.append( Item(channel=__channel__, action="findvideos", title=scrapedtitle, url=scrapedurl , thumbnail=scrapedthumbnail+"&w=300&h=400&zc=1&ft=jpg" , plot=scrapedplot , folder=True) )

    # Extrae el paginador
    patronvideos  = '
>>> Post vecchi >>>'
    matches = re.compile(patronvideos,re.DOTALL).findall(data)
    scrapertools.printMatches(matches)

    if len(matches)>0:
        scrapedurl = urlparse.urljoin(item.url,matches[0])
        itemlist.append( Item(channel=__channel__, action="peliculas", title="[COLOR orange]Avanti >>[/COLOR]" , url=scrapedurl , folder=True) )

    return itemlist

def test():
    from servers import servertools

    # mainlist
    mainlist_items = mainlist(Item())
    # Da por bueno el canal si alguno de los videos de "Novedades" devuelve mirrors
    novedades_items = peliculas(mainlist_items[0])
    bien = False
    for novedades_item in novedades_items:
        mirrors = servertools.find_video_items( item=novedades_item )
        if len(mirrors)>0:
            bien = True
            break

    return bien


# train_model.py
import os
import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from transformer_autoencoder import TransformerAutoencoder
from mel_spectrogram_dataset import MelSpectrogramDataset
from arg_parser import parse_args
from batch_train_loop import train_epoch

def main():
    args = parse_args()

    if args.use_mps and torch.backends.mps.is_available():
        device = torch.device('mps')
    elif args.use_cuda and torch.cuda.is_available():
        device = torch.device('cuda')
    else:
        device = torch.device('cpu')

    print(f"Using device: {device}")

    d_model = 128
    mel_train_dataset = MelSpectrogramDataset(n_mels=d_model)

    print(f"Total number of training data examples: {len(mel_train_dataset)}")

    train_loader = DataLoader(mel_train_dataset, batch_size=args.batch_size,
                              shuffle=True, collate_fn=mel_train_dataset.collate_fn)

    # Define the bottleneck dimension
    bottleneck_dim = 128

    # Instantiate the TransformerAutoencoder with the bottleneck_dim parameter
    model = TransformerAutoencoder(d_model=d_model, nhead=4,
                                   num_layers=4, dim_feedforward=512,
                                   bottleneck_dim=bottleneck_dim, dropout=0.5).to(device)
    optimizer = optim.Adam(model.parameters(), lr=args.lr)
    # Define the learning rate scheduler
    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', verbose=True, patience=5)

    start_epoch = 0
    if args.checkpoint and os.path.isfile(args.checkpoint):
        checkpoint = torch.load(args.checkpoint)
        model.load_state_dict(checkpoint['model_state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        start_epoch = checkpoint['epoch']

    num_epochs = args.epochs + start_epoch
    for epoch in range(start_epoch, num_epochs):
        avg_loss = train_epoch(model, train_loader, optimizer, device)
        print(f'Epoch [{epoch + 1}/{num_epochs}], Average Loss: {avg_loss:.4f}')

        # Step the learning rate scheduler
        scheduler.step(avg_loss)

        # Save checkpoint
        torch.save({
            'epoch': epoch + 1,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
        }, f'checkpoint_epoch_{epoch + 1}.pth')

if __name__ == '__main__':
    main()


from pyspark.sql import SparkSession

spark = SparkSession.builder.master('local').getOrCreate()
#df = spark.range(1000).toDF("number")
#divisBy2 = df.where("number % 2 = 0")
#print(divisBy2.count())
carrier_data = spark\
    .read\
    .option("inferSchema","true")\
    .option("delimiter",",")\
    .option("header","true")\
    .csv("D:\Santhu_practice\datasets\carriers (2).csv")
carrier_data.show(truncate=False)

print(carrier_data.take(10))


"""
OSQuery process_open_sockets ORM
"""
from ...orm import BaseModel
from peewee import BigIntegerField, ForeignKeyField, IntegerField, TextField
from .etc_protocols import EtcProtocols
from ..consts import SocketFamily

class ProcessOpenSockets(BaseModel):
    """
    Processes which have open network sockets on the system.
    Examples:
        select * from process_open_sockets where pid = 1
    """
    pid = IntegerField(help_text="Process (or thread) ID")  # {'index': True}
    fd = BigIntegerField(help_text="Socket file descriptor number")
    socket = BigIntegerField(help_text="Socket handle or inode number")
    # family = IntegerField(help_text="Network protocol (IPv4, IPv6)")
    family = IntegerField(choices=SocketFamily, help_text="Network protocol (IPv4, IPv6)")
    # protocol = IntegerField(help_text="Transport protocol (TCP/UDP)")
    protocol = ForeignKeyField(EtcProtocols, EtcProtocols.number, column_name="protocol", help_text="Transport protocol (TCP/UDP)")
    local_address = TextField(help_text="Socket local address")
    remote_address = TextField(help_text="Socket remote address")
    local_port = IntegerField(help_text="Socket local port")
    remote_port = IntegerField(help_text="Socket remote port")
    path = TextField(help_text="For UNIX sockets (family=AF_UNIX), the domain path")

    class Meta:
        table_name = "process_open_sockets"

# OS specific properties for Linux_MacOS_Windows
class Linux_MacOS_Windows_ProcessOpenSockets(ProcessOpenSockets):
    state = TextField(help_text="TCP socket state")

    class Meta:
        table_name = "process_open_sockets"

# OS specific properties for Linux
class Linux_ProcessOpenSockets(Linux_MacOS_Windows_ProcessOpenSockets):
    net_namespace = TextField(help_text="The inode number of the network namespace")

    class Meta:
        table_name = "process_open_sockets"


# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

# https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an \"AS IS\" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import urllib.request
import os
import zipfile
import random
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers
from tensorflow.keras import Model
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.optimizers import RMSprop
from shutil import copyfile

data_url = "https://download.microsoft.com/download/3/E/1/3E1C3F21-ECDB-4869-8368-6DEBA77B919F/kagglecatsanddogs_3367a.zip"
data_file_name = "catsdogs.zip"
download_dir = 'tmp/'
urllib.request.urlretrieve(data_url, data_file_name)
zip_ref = zipfile.ZipFile(data_file_name, 'r')
zip_ref.extractall(download_dir)
zip_ref.close()

print(len(os.listdir('tmp/PetImages/Cat/')))
print(len(os.listdir('tmp/PetImages/Dog/')))

try:
    os.mkdir('tmp/cats-v-dogs')
    os.mkdir('tmp/cats-v-dogs/training')
    os.mkdir('tmp/cats-v-dogs/testing')
    os.mkdir('tmp/cats-v-dogs/training/cats')
    os.mkdir('tmp/cats-v-dogs/training/dogs')
    os.mkdir('tmp/cats-v-dogs/testing/cats')
    os.mkdir('tmp/cats-v-dogs/testing/dogs')
except OSError:
    pass

import random
from shutil import copyfile

def split_data(SOURCE, TRAINING, TESTING, SPLIT_SIZE):
    files = []
    for filename in os.listdir(SOURCE):
        file = SOURCE + filename
        if os.path.getsize(file) > 0:
            files.append(filename)
        else:
            print(filename + " is zero length, so ignoring.")

    training_length = int(len(files) * SPLIT_SIZE)
    testing_length = int(len(files) - training_length)
    shuffled_set = random.sample(files, len(files))
    training_set = shuffled_set[0:training_length]
    testing_set = shuffled_set[:testing_length]

    for filename in training_set:
        this_file = SOURCE + filename
        destination = TRAINING + filename
        copyfile(this_file, destination)

    for filename in testing_set:
        this_file = SOURCE + filename
        destination = TESTING + filename
        copyfile(this_file, destination)

CAT_SOURCE_DIR = "tmp/PetImages/Cat/"
TRAINING_CATS_DIR = "tmp/cats-v-dogs/training/cats/"
TESTING_CATS_DIR = "tmp/cats-v-dogs/testing/cats/"
DOG_SOURCE_DIR = "tmp/PetImages/Dog/"
TRAINING_DOGS_DIR = "tmp/cats-v-dogs/training/dogs/"
TESTING_DOGS_DIR = "tmp/cats-v-dogs/testing/dogs/"

split_size = .9
split_data(CAT_SOURCE_DIR, TRAINING_CATS_DIR, TESTING_CATS_DIR, split_size)
split_data(DOG_SOURCE_DIR, TRAINING_DOGS_DIR, TESTING_DOGS_DIR, split_size)

# Expected output
# 666.jpg is zero length, so ignoring
# 11702.jpg is zero length, so ignoring

print(len(os.listdir('tmp/cats-v-dogs/training/cats/')))
print(len(os.listdir('tmp/cats-v-dogs/training/dogs/')))
print(len(os.listdir('tmp/cats-v-dogs/testing/cats/')))
print(len(os.listdir('tmp/cats-v-dogs/testing/dogs/')))

TRAINING_DIR = "tmp/cats-v-dogs/training/"
# Experiment with your own parameters here to really try to drive it to 99.9% accuracy or better
train_datagen = ImageDataGenerator(rescale=1./255,
                                   rotation_range=40,
                                   width_shift_range=0.2,
                                   height_shift_range=0.2,
                                   shear_range=0.2,
                                   zoom_range=0.2,
                                   horizontal_flip=True,
                                   fill_mode='nearest')
train_generator = train_datagen.flow_from_directory(TRAINING_DIR,
                                                    batch_size=100,
                                                    class_mode='binary',
                                                    target_size=(150, 150))

VALIDATION_DIR = "tmp/cats-v-dogs/testing/"
# Experiment with your own parameters here to really try to drive it to 99.9% accuracy or better
validation_datagen = ImageDataGenerator(rescale=1./255)
validation_generator = validation_datagen.flow_from_directory(VALIDATION_DIR,
                                                              batch_size=100,
                                                              class_mode='binary',
                                                              target_size=(150, 150))

weights_url = "https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5"
weights_file = "inception_v3.h5"
urllib.request.urlretrieve(weights_url, weights_file)

pre_trained_model = InceptionV3(input_shape=(150, 150, 3),
                                include_top=False,
                                weights=None)

pre_trained_model.load_weights(weights_file)

for layer in pre_trained_model.layers:
    layer.trainable = False

# pre_trained_model.summary()

last_layer = pre_trained_model.get_layer('mixed7')
print('last layer output shape: ', last_layer.output_shape)
last_output = last_layer.output

# Flatten the output layer to 1 dimension
x = layers.Flatten()(last_output)
# Add a fully connected layer with 1,024 hidden units and ReLU activation
x = layers.Dense(1024, activation='relu')(x)
# Add a dropout rate of 0.2
x = layers.Dropout(0.3)(x)
# Add a final sigmoid layer for classification
x = layers.Dense(1, activation='sigmoid')(x)

model = Model(pre_trained_model.input, x)

model.compile(optimizer=RMSprop(lr=0.0001),
              loss='binary_crossentropy',
              metrics=['acc'])

history = model.fit_generator(
            train_generator,
            validation_data=validation_generator,
            epochs=20,
            verbose=1)

model.save("catsdogs.h5")


from keras.optimizers import SGD
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TerminateOnNaN, CSVLogger
from keras import backend as K
from keras.models import load_model
from math import ceil
import numpy as np
from matplotlib import pyplot as plt
from models.keras_ssd_face import build_ssd_face
from keras_loss_function.keras_ssd_loss import SSDLoss
from keras_layers.keras_layer_AnchorBoxes import AnchorBoxes
from keras_layers.keras_layer_DecodeDetections import DecodeDetections
from keras_layers.keras_layer_DecodeDetectionsFast import DecodeDetectionsFast
from keras_layers.keras_layer_L2Normalization import L2Normalization

from ssd_encoder_decoder.ssd_input_encoder import SSDInputEncoder
from ssd_encoder_decoder.ssd_output_decoder import decode_detections, decode_detections_fast

from data_generator.object_detection_2d_data_generator import DataGenerator
from data_generator.object_detection_2d_geometric_ops import Resize
from data_generator.object_detection_2d_photometric_ops import ConvertTo3Channels
from data_generator.data_augmentation_chain_original_ssd import SSDDataAugmentation
from data_generator.object_detection_2d_misc_utils import apply_inverse_transforms

# Set up the model configurations
batch_size = 16
img_height = 640 # Height of the model input images
img_width = 640 # Width of the model input images
img_channels = 3 # Number of color channels of the model input images
mean_color = [123, 117, 104] # The per-channel mean of the images in the dataset. Do not change this value if you're using any of the pre-trained weights.
swap_channels = [2, 1, 0] # The color channel order in the original SSD is BGR, so we'll have the model reverse the color channel order of the input images.
n_classes = 1 # Number of positive classes, e.g. 1 face
scales_s3fd = [0.03, 0.05, 0.10, 0.20, 0.40, 0.80, 0.81] # The anchor boxes ratio per each layer (16, 32, 64, 128, 256, 512)
scales = scales_s3fd
aspect_ratios = [[1.0],[1.0],[1.0],[1.0],[1.0],[1.0]] # The anchor box aspect ratios used in the original SSD300; the order matters
two_boxes_for_ar1 = False
steps = [4, 8, 16, 32, 64, 128] # The space between two adjacent anchor box center points for each predictor layer.
offsets = [0.5, 0.5, 0.5, 0.5, 0.5, 0.5] # The offsets of the first anchor box center points from the top and left borders of the image as a fraction of the step size for each predictor layer.
clip_boxes = False # Whether or not to clip the anchor boxes to lie entirely within the image boundaries
variances = [0.1, 0.1, 0.2, 0.2] # The variances by which the encoded target coordinates are divided as in the original implementation
normalize_coords = True

# 1: Build the Keras model.
K.clear_session() # Clear previous models from memory.

model = build_ssd_face(image_size=(img_height, img_width, img_channels),
                n_classes=n_classes,
                mode='training',
                l2_regularization=0.0005,
                scales=scales,
                aspect_ratios_per_layer=aspect_ratios,
                two_boxes_for_ar1=two_boxes_for_ar1,
                steps=steps,
                offsets=offsets,
                clip_boxes=clip_boxes,
                variances=variances,
                normalize_coords=normalize_coords,
                subtract_mean=mean_color,
                swap_channels=swap_channels)

# 2: Load some weights into the model.

# TODO: Set the path to the weights you want to load.

weights_path = '/home/ubuntu/Result/ssd_face/ssd_face_initial_weights.h5'
model.load_weights(weights_path, by_name=True)

# 3: Instantiate an optimizer and the SSD loss function and compile the model.
sgd = SGD(lr=0.001,momentum=0.9,decay=0.0005)
ssd_loss = SSDLoss(neg_pos_ratio=3, alpha=1.0)
model.compile(optimizer=sgd, loss=ssd_loss.compute_loss)

# 1: Instantiate two `DataGenerator` objects: One for training, one for validation.

# Optional: If you have enough memory, consider loading the images into memory for the reasons explained above.

train_dataset = DataGenerator(load_images_into_memory=True, hdf5_dataset_path=None)
val_dataset = DataGenerator(load_images_into_memory=True, hdf5_dataset_path=None)

# 2: Parse the image and label lists for the training and validation datasets.

# TODO: Set the paths to your dataset here.

# Images
images_dir = '/home/ubuntu/Datasets/WiderFace/'

# Ground truth
train_labels_filename = '/home/ubuntu/Datasets/WiderFace/labels_train.csv'
val_labels_filename   = '/home/ubuntu/Datasets/WiderFace/labels_val.csv'

train_dataset.parse_csv(images_dir=images_dir,
                        labels_filename=train_labels_filename,
                        input_format=['image_name', 'class_id', 'xmin', 'xmax', 'ymin', 'ymax'], # This is the order of the first six columns in the CSV file that contains the labels for your dataset. If your labels are in XML format, maybe the XML parser will be helpful, check the documentation.
                        include_classes='all')

val_dataset.parse_csv(images_dir=images_dir,
                      labels_filename=val_labels_filename,
                      input_format=['image_name', 'class_id', 'xmin', 'xmax', 'ymin', 'ymax'],
                      include_classes='all')

# Get the number of samples in the training and validations datasets.
train_dataset_size = train_dataset.get_dataset_size()
val_dataset_size   = val_dataset.get_dataset_size()

print("Number of images in the training dataset:\t{:>6}".format(train_dataset_size))
print("Number of images in the validation dataset:\t{:>6}".format(val_dataset_size))

# 3: Set the batch size.
batch_size = batch_size # Change the batch size if you like, or if you run into GPU memory issues.

# 4: Set the image transformations for pre-processing and data augmentation options.

# For the training generator:
ssd_data_augmentation = SSDDataAugmentation(img_height=img_height,
                                            img_width=img_width,
                                            background=mean_color)

# For the validation generator:
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height, width=img_width)

# 5: Instantiate an encoder that can encode ground truth labels into the format needed by the SSD loss function.

# The encoder constructor needs the spatial dimensions of the model's predictor layers to create the anchor boxes.
predictor_sizes = [model.get_layer('conv3_3_norm_mbox_conf').output_shape[1:3],
                   model.get_layer('conv4_3_norm_mbox_conf').output_shape[1:3],
                   model.get_layer('conv5_3_norm_mbox_conf').output_shape[1:3],
                   model.get_layer('fc7_mbox_conf').output_shape[1:3],
                   model.get_layer('conv6_2_mbox_conf').output_shape[1:3],
                   model.get_layer('conv7_2_mbox_conf').output_shape[1:3]]

ssd_input_encoder = SSDInputEncoder(img_height=img_height,
                                    img_width=img_width,
                                    n_classes=n_classes,
                                    predictor_sizes=predictor_sizes,
                                    scales=scales,
                                    aspect_ratios_per_layer=aspect_ratios,
                                    two_boxes_for_ar1=two_boxes_for_ar1,
                                    steps=steps,
                                    offsets=offsets,
                                    clip_boxes=clip_boxes,
                                    variances=variances,
                                    matching_type='multi',
                                    pos_iou_threshold=0.35,
                                    neg_iou_limit=0.35,
                                    normalize_coords=normalize_coords)

# 6: Create the generator handles that will be passed to Keras' `fit_generator()` function.

train_generator = train_dataset.generate(batch_size=batch_size,
                                         shuffle=True,
                                         transformations=[ssd_data_augmentation],
                                         label_encoder=ssd_input_encoder,
                                         returns={'processed_images',
                                                  'encoded_labels'},
                                         keep_images_without_gt=False)

val_generator = val_dataset.generate(batch_size=batch_size,
                                     shuffle=False,
                                     transformations=[convert_to_3_channels,
                                                      resize],
                                     label_encoder=ssd_input_encoder,
                                     returns={'processed_images',
                                              'encoded_labels'},
                                     keep_images_without_gt=False)

# Get the number of samples in the training and validations datasets.
train_dataset_size = train_dataset.get_dataset_size()
val_dataset_size   = val_dataset.get_dataset_size()

print("Number of images in the training dataset:\t{:>6}".format(train_dataset_size))
print("Number of images in the validation dataset:\t{:>6}".format(val_dataset_size))

# Define a learning rate schedule.

def lr_schedule(epoch):
    if epoch < 200:
        return 0.001
    elif epoch < 250:
        return 0.0001
    else:
        return 0.00001

# Define model callbacks.

# TODO: Set the filepath under which you want to save the model.
save_dir = "/home/ubuntu/Result/ssd_face/"
model_checkpoint = ModelCheckpoint(filepath= save_dir + 'ssd_face_epoch-{epoch:02d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.h5',
                                   monitor='val_loss',
                                   verbose=1,
                                   save_weights_only=True,
                                   mode='auto',
                                   period=1)

csv_logger = CSVLogger(filename=save_dir + 'ssd_face_training_log.csv',
                       separator=',',
                       append=True)

learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule,
                                                verbose=1)

terminate_on_nan = TerminateOnNaN()

callbacks = [model_checkpoint,
             csv_logger,
             learning_rate_scheduler,
             terminate_on_nan]

## TRAINNING
# If you're resuming a previous training, set `initial_epoch` and `final_epoch` accordingly.
initial_epoch   = 0
final_epoch     = 300
steps_per_epoch = ceil(train_dataset_size/batch_size)

history = model.fit_generator(generator=train_generator,
                              steps_per_epoch=steps_per_epoch,
                              epochs=final_epoch,
                              callbacks=callbacks,
                              validation_data=val_generator,
                              validation_steps=ceil(val_dataset_size/batch_size),
                              initial_epoch=initial_epoch)


import re
def inspect_fib_all(output):
    """
    Parse the output of the 'inspect fib all' command.

    Args:
        output (str): The output of the 'inspect fib all' command.

    Returns:
        dict: Parsed data containing information about paths, prefixes, and VPNs.
    """
    parsed_data = {}
    try:
        lines = re.split(r'[\n,]', output)
        current_path = None

        for line in lines:
            line = line.strip()
            if line.startswith("Path ID:"):
                # Start of a new path
                if current_path is not None:
                    parsed_data[current_path['Path ID']] = current_path
                current_path = {}
                path_id = line.split(': ')[1].strip()
                current_path['Path ID'] = path_id
            elif line.startswith("Path Type:"):
                path_type = line.split(': ')[1].strip()
                current_path['Path Type'] = path_type
            elif line.startswith("Vpn Type:"):
                vpn_type = line.split(': ')[1].strip()
                current_path['Vpn Type'] = vpn_type
            elif line.startswith("Site ID:"):
                site_id = line.split(': ')[1].strip()
                current_path['Site ID'] = site_id
            elif line.startswith("Site Name:"):
                site_name = line.split(': ')[1].strip()
                current_path['Site Name'] = site_name
            elif line.startswith("Status:"):
                status = line.split(': ')[1].strip()
                current_path['Status'] = status.lower() == 'true'
            elif line.startswith("Path Info:"):
                path_info = line.split(': ')[1].strip()
                current_path['Path Info'] = path_info
            elif line.startswith("Prefixes:") or line.startswith("V6 Prefixes:"):
                # Skip the header for prefixes and v6 prefixes
                continue
            elif line.startswith("Prefix:"):
                if 'Prefixes' not in current_path:
                    current_path['Prefixes'] = []
                prefix = line.split(': ')[1].strip()
                current_path['Prefixes'].append(prefix)
            elif line.startswith("V6 Prefix:"):
                if 'V6 Prefixes' not in current_path:
                    current_path['V6 Prefixes'] = []
                v6_prefix = line.split(': ')[1].strip()
                current_path['V6 Prefixes'].append(v6_prefix)

        # Add the last path to the parsed data
        if current_path is not None:
            parsed_data[current_path['Path ID']] = current_path
    except Exception as e:
        print(f"Error occurred while parsing 'inspect fib all' output: {str(e)}")

    return parsed_data