from nlp import NLP
from analysis import Analysis

class Main:
    '''
        Explanation: Driver class which handles all other functions and classes

        Class Variables:
            sent_en: Raw text from source file in English
            sent_hi: Raw text from source file in Hindi
            lang_en: Represents English
            lang_hi: Represents Hindi
    '''

    def __init__(self):
        '''
            Explanation: Initialisation function, used to initialise class variables
        '''
        self.sent_en = self.get_sent('data/english.txt')
        self.sent_hi = self.get_sent('data/hindi.txt')
        self.lang_en = 'en'
        self.lang_hi = 'hi'

    def get_sent(self, fileName):
        '''
            Explanation: Given a file, it reads it and returns it as a list of strings line by line

            Parameters:
                fileName: Name of the file

            Return:
                sentences: List of string
        '''
        f = open(fileName, 'r')
        lines = f.readlines()
        sentences = []
        for line in lines:
            sentences.append(line.strip())

        return sentences

    def get_json(self, lang, sentences):
        '''
            Explanation: Creates object of class NLP to get the list of sentences in appropriate tagged JSON format

            Parameters:
                lang: Language of the list of sentences
                sentences: List of string represeting the sentences

            Return:
                JSON object containing annotated values
        '''
        nlp = NLP(lang, sentences)
        return nlp.get_dependencies()

    def main(self):
        '''
            Explanation: Driver function used to process sentences and send it for analysis.
        '''
        parse_en = self.get_json(self.lang_en, self.sent_en)
        parse_hi = self.get_json(self.lang_hi, self.sent_hi)
        analysis = Analysis(parse_en, parse_hi)
        analysis.drive()

# Driver code
m = Main()
m.main()


from pywebio import *
from pywebio.output import *
from pywebio.input import *
from pywebio.pin import *
from pywebio.session import hold

def put_pin_value(text):
    with use_scope('text_output', clear=True):
        put_text(text)

def main():
    put_table([
                ['Commodity', 'Price / unit'],
                ['Apple', '0.5'],
                ['Banana', '0.4'],
                ['Avacado', '1.2'],
            ])

    put_tabs([
        {'title': 'Search by fruit', 'content': [
            put_row(
                [put_input('fruit'),
                put_buttons(['search'], lambda _: put_pin_value(pin.fruit)),
            ], )
        ]},
        {'title': 'Search by price', 'content': [
            put_row(
                [put_input('price'),
                put_buttons(['search'], lambda _: put_pin_value(pin.price)),
            ], )
        ]},
        {'title': 'Help', 'content': 'Input a fruit name of interest then hit the search button.'},
    ])

    use_scope('text_output')
    hold()


"""Very basic tests to check that nothing crashes. Not 100% coverage but covers what we care about."""

from argparse import ArgumentParser
from copy import deepcopy
import os
import sys
from tempfile import TemporaryDirectory, NamedTemporaryFile
import unittest

sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

from chemprop.data.utils import get_data
from chemprop.features import clear_cache, get_available_features_generators
from chemprop.parsing import add_train_args, modify_train_args, add_predict_args, modify_predict_args
from chemprop.train import cross_validate, make_predictions
from chemprop.utils import create_logger
from hyperparameter_optimization import grid_search
from scripts.avg_dups import average_duplicates
from scripts.overlap import overlap
from scripts.save_features import generate_and_save_features
from scripts.similarity import scaffold_similarity, morgan_similarity

# TODO hyperopt

class TestScripts(unittest.TestCase):
    def tearDown(self):
        clear_cache()

    def test_avg_dups(self):
        try:
            parser = ArgumentParser()
            parser.add_argument('--data_path', type=str,
                                help='Path to data CSV file')
            parser.add_argument('--save_path', type=str,
                                help='Path where average data CSV file will be saved')
            args = parser.parse_args([])
            args.data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            args.save_path = NamedTemporaryFile().name
            average_duplicates(args)
            os.remove(args.save_path)
        except:
            self.fail('avg_dups')

    def test_overlap(self):
        try:
            parser = ArgumentParser()
            parser.add_argument('--data_path_1', type=str,
                                help='Path to first data CSV file')
            parser.add_argument('--data_path_2', type=str,
                                help='Path to second data CSV file')
            parser.add_argument('--compound_names_1', action='store_true', default=False,
                                help='Whether data_path_1 has compound names in addition to smiles')
            parser.add_argument('--compound_names_2', action='store_true', default=False,
                                help='Whether data_path_2 has compound names in addition to smiles')
            parser.add_argument('--save_intersection_path', type=str, default=None,
                                help='Path to save intersection at; labeled with data_path 1 header')
            parser.add_argument('--save_difference_path', type=str, default=None,
                                help='Path to save molecules in dataset 1 that are not in dataset 2; labeled with data_path 1 header')
            args = parser.parse_args([])
            args.data_path_1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            args.data_path_2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')

            overlap(args)
        except:
            self.fail('overlap')

    def test_save_features(self):
        try:
            parser = ArgumentParser()
            parser.add_argument('--data_path', type=str,
                                help='Path to data CSV')
            parser.add_argument('--features_generator', type=str,
                                choices=get_available_features_generators(),
                                help='Type of features to generate')
            parser.add_argument('--save_path', type=str,
                                help='Path to .npz file where features will be saved as a numpy compressed archive')
            parser.add_argument('--save_frequency', type=int, default=10000,
                                help='Frequency with which to save the features')
            parser.add_argument('--restart', action='store_true', default=False,
                                help='Whether to not load partially complete featurization and instead start from scratch')
            parser.add_argument('--max_data_size', type=int,
                                help='Maximum number of data points to load')
            parser.add_argument('--sequential', action='store_true', default=False,
                                help='Whether to run sequentially rather than in parallel')
            args = parser.parse_args([])
            args.data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            args.save_path = NamedTemporaryFile(suffix='.npz').name
            args.restart = True
            args.features_generator = 'morgan_count'

            generate_and_save_features(args)

            os.remove(args.save_path)
        except:
            self.fail('save_features')

    def test_similarity(self):
        try:
            parser = ArgumentParser()
            parser.add_argument('--data_path_1', type=str,
                                help='Path to first data CSV file')
            parser.add_argument('--data_path_2', type=str,
                                help='Path to second data CSV file')
            parser.add_argument('--compound_names_1', action='store_true', default=False,
                                help='Whether data_path_1 has compound names in addition to smiles')
            parser.add_argument('--compound_names_2', action='store_true', default=False,
                                help='Whether data_path_2 has compound names in addition to smiles')
            parser.add_argument('--radius', type=int, default=3,
                                help='Radius of Morgan fingerprint')
            parser.add_argument('--sample_rate', type=float, default=1.0,
                                help='Rate at which to sample pairs of molecules for Morgan similarity (to reduce time)')
            args = parser.parse_args([])

            args.data_path_1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            args.data_path_2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')

            data_1 = get_data(path=args.data_path_1, use_compound_names=args.compound_names_1)
            data_2 = get_data(path=args.data_path_2, use_compound_names=args.compound_names_2)

            scaffold_similarity(data_1.smiles(), data_2.smiles())
            morgan_similarity(data_1.smiles(), data_2.smiles(), args.radius, args.sample_rate)
        except:
            self.fail('similarity')

class TestTrain(unittest.TestCase):
    def setUp(self):
        parser = ArgumentParser()
        add_train_args(parser)
        args = parser.parse_args([])
        args.data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
        args.dataset_type = 'regression'
        args.batch_size = 2
        args.hidden_size = 5
        args.epochs = 1
        args.quiet = True
        self.args = args
        logger = create_logger(name='train', save_dir=args.save_dir, quiet=args.quiet)
        self.logger = logger

    def tearDown(self):
        self.args = None
        self.logger = None
        clear_cache()

    def test_regression_default(self):
        try:
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('regression_default')

    def test_classification_multiclass_default(self):
        try:
            self.args.data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tox21_toy.csv')
            self.args.dataset_type = 'classification'
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('classification_default')

    def test_rdkit_2d_features(self):
        try:
            self.args.features_generator = ['rdkit_2d_normalized']
            self.args.no_features_scaling = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('rdkit_2d_features')

    def test_rdkit_2d_features_unnormalized(self):
        try:
            self.args.features_generator = ['rdkit_2d']
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('rdkit_2d_features_unnormalized')

    def test_features_only(self):
        try:
            self.args.features_generator = ['morgan']
            self.features_only = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('features_only')

    def test_checkpoint(self):
        try:
            args_copy = deepcopy(self.args)
            temp_dir = TemporaryDirectory()
            self.args.save_dir = temp_dir.name
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
            args_copy.checkpoint_dir = temp_dir.name
            args_copy.test = True
            modify_train_args(args_copy)
            cross_validate(args_copy, self.logger)
        except:
            self.fail('checkpoint')

    def test_save_smiles_splits(self):
        try:
            self.args.save_smiles_splits = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('save smiles splits')

    def test_scaffold(self):
        try:
            self.args.split_type = 'scaffold_balanced'
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('scaffold')

    def test_no_cuda(self):
        try:
            self.args.no_cuda = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('no_cuda')

    def test_show_individual_scores(self):
        try:
            self.args.show_individual_scores = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('show_individual_scores')

    def test_no_cache(self):
        try:
            self.args.no_cache = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('no_cache')

    def test_num_folds_ensemble(self):
        try:
            self.args.num_folds = 2
            self.args.ensemble_size = 2
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('num_folds_ensemble')

    def test_bias(self):
        try:
            self.args.bias = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('bias')

    def test_activation_prelu(self):
        try:
            self.args.activation = 'PReLU'
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('activation_prelu')

    def test_undirected_messages(self):
        try:
            self.args.undirected = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('undirected_messages')

    def test_atom_messages(self):
        try:
            self.args.atom_messages = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('atom_messages')

    def test_features_path(self):
        try:
            self.args.features_path = [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_features.npz')]
            self.args.no_features_scaling = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('features_path')

    def test_separate_val_test(self):
        try:
            self.args.separate_val_set = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            self.args.separate_test_set = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            self.args.features_path = [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_features.npz')]
            self.args.separate_val_set_features = [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_features.npz')]
            self.args.separate_test_set_features = [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_features.npz')]
            self.args.no_features_scaling = True
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('separate_val_test')

    def test_predetermined_split(self):
        try:
            self.args.split_type = 'predetermined'
            self.args.folds_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_folds.pkl')
            self.args.val_fold_index = 1
            self.args.test_fold_index = 2
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('predetermined_split')

    def test_config(self):
        try:
            self.args.config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json')
            modify_train_args(self.args)
            cross_validate(self.args, self.logger)
        except:
            self.fail('config')

class TestPredict(unittest.TestCase):
    def setUp(self):
        parser = ArgumentParser()
        add_train_args(parser)
        args = parser.parse_args([])
        args.data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
        args.dataset_type = 'regression'
        args.batch_size = 2
        args.hidden_size = 5
        args.epochs = 1
        args.quiet = True
        self.temp_dir = TemporaryDirectory()
        args.save_dir = self.temp_dir.name
        logger = create_logger(name='train', save_dir=args.save_dir, quiet=args.quiet)
        modify_train_args(args)
        cross_validate(args, logger)
        clear_cache()

        parser = ArgumentParser()
        add_predict_args(parser)
        args = parser.parse_args([])
        args.batch_size = 2
        args.checkpoint_dir = self.temp_dir.name
        args.preds_path = NamedTemporaryFile().name
        args.test_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_smiles.csv')
        self.args = args

    def tearDown(self):
        self.temp_dir.cleanup()
        os.remove(self.args.preds_path)
        self.args = None
        clear_cache()

    def test_predict(self):
        try:
            modify_predict_args(self.args)
            make_predictions(self.args)
        except:
            self.fail('predict')

    def test_predict_compound_names(self):
        try:
            self.args.test_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy_smiles_names.csv')
            self.args.compound_names = True
            modify_predict_args(self.args)
            make_predictions(self.args)
        except:
            self.fail('predict_compound_names')

class TestHyperopt(unittest.TestCase):
    def test_hyperopt(self):
        try:
            parser = ArgumentParser()
            add_train_args(parser)
            parser.add_argument('--num_iters', type=int, default=20,
                                help='Number of hyperparameter choices to try')
            parser.add_argument('--config_save_path', type=str,
                                help='Path to .json file where best hyperparameter settings will be written')
            parser.add_argument('--log_dir', type=str,
                                help='(Optional) Path to a directory where all results of the hyperparameter optimization will be written')
            args = parser.parse_args([])
            args.data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'delaney_toy.csv')
            args.dataset_type = 'regression'
            args.batch_size = 2
            args.hidden_size = 5
            args.epochs = 1
            args.quiet = True
            temp_file = NamedTemporaryFile()
            args.config_save_path = temp_file.name
            args.num_iters = 3
            modify_train_args(args)

            grid_search(args)
            clear_cache()
        except:
            self.fail('hyperopt')

if __name__ == '__main__':
    unittest.main()


import pandas as pd
import codecs
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
import streamlit as st

st.title('SMS SPAM DETECTION')

with codecs.open('spam.csv','r',encoding='utf-8',errors='ignore') as f:
    dataset=pd.read_csv(f)

dataset=dataset.drop(['Unnamed: 2','Unnamed: 3','Unnamed: 4'],axis=1)
dataset=dataset.rename(columns={'v1':'label','v2':'msg'})
dataset['length']=dataset['msg'].apply(len)
dataset['output']=dataset.label.map({'ham':0,'spam':1})

x=dataset.msg
x=x.values

y=dataset.output
y=y.values

x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2)

vect=CountVectorizer()
x_train=vect.fit_transform(x_train)
x_test=vect.transform(x_test)

classifier1=LogisticRegression(solver='liblinear',penalty='l1')
classifier1.fit(x_train,y_train)

k=st.text_input('Enter Message')
if st.button('Classify SMS'):
    data=[k]
    data=vect.transform(data)
    result=classifier1.predict(data)
    if result==0:
        st.success('Not A Spam')
    else:
        st.error('Spam Message')


# -*- coding: utf-8 -*-

import dominate
import os
import json

from .common import navbar
from .utils import AbedHTMLTypes, copy_data_file
from ..conf import settings
from ..io import info
from ..utils import clean_str
from ..results.models import AbedTableTypes

tags = dominate.tags

def create_rank_graphs_html(tables):
    html = generate_graphs_html(tables)
    write_tables_html(html)

def get_table_id(table):
    values = []
    values.append(clean_str(table.target))
    values.append(clean_str(table.type))
    if table.is_metric:
        if settings.TYPE == "ASSESS":
            values.append(clean_str(table.metricname))
        elif settings.TYPE == "CV_TT":
            values.append(clean_str(table.trainmetricname))
            values.append(clean_str(table.testmetricname))
    if table.is_summary:
        values.append("summary")
    return "_".join(values)

def generate_graphs_html(tables):
    doc = dominate.document(title="ABED Rank Graphs")

    with doc.head:
        # meta tags
        tags.meta(charset="utf-8")
        tags.meta(http_equiv="X-UA-Compatible", content="IE=edge")
        tags.meta(name="viewport", content="width=device-width, initial-scale=1")
        tags.meta(name="description", content="Rank graphs for ABED results")
        tags.meta(name="author", content="Gertjan van den Burg")

        # JQuery & Bootstrap JS
        tags.script(
            type="text/javascript",
            src=copy_data_file("jquery/1.11.3/jquery.min.js"),
        )
        tags.script(
            type="text/javascript",
            src=copy_data_file("bootstrap-3.3.5-dist/js/bootstrap.min.js"),
        )
        # Bootstrap core CSS
        tags.link(
            rel="stylesheet",
            href=copy_data_file("bootstrap-3.3.5-dist/css/bootstrap.min.css"),
        )

        # D3 and Labella
        tags.script(type="text/javascript", src=copy_data_file("d3/d3.min.js"))
        tags.script(type="text/javascript", src=copy_data_file("d3kit/d3kit.min.js"))
        tags.script(
            type="text/javascript",
            src=copy_data_file("labella/labella.min.js"),
        )
        tags.script(
            type="text/javascript",
            src=copy_data_file("d3kit-timeline/d3kit-timeline.min.js"),
        )

        # ABED CSS & JS
        tags.link(rel="stylesheet", href=copy_data_file("abed/css/abed.css"))
        tags.script(
            type="text/javascript",
            src=copy_data_file("abed/js/abed_rankgraphs.js"),
        )

    tables = [t for t in tables if t.is_summary and t.type == AbedTableTypes.RANKS]
    labels = sorted(set([t.target for t in tables]))
    lbl_table = {}
    for lbl in labels:
        lbl_table[lbl] = [t for t in tables if t.target == lbl]

    with doc:
        with tags.div(id="header"):
            navbar(active=AbedHTMLTypes.RANK_GRAPHS)
        with tags.div(id="content"):
            with tags.div(_class="container"):
                for lbl in sorted(lbl_table.keys()):
                    with tags.div(_class="panel panel-default"):
                        with tags.div(_class="panel-heading"):
                            tags.h3("Target: %s" % lbl, _class="panel-title")
                        with tags.div(_class="panel-body"):
                            for table in lbl_table[lbl]:
                                tabid = get_table_id(table)
                                if table.is_metric:
                                    if settings.TYPE == "ASSESS":
                                        tags.p("Metric: %s" % table.metricname)
                                    elif settings.TYPE == "CV_TT":
                                        with tags.div(_class="col-xs-6"):
                                            tags.b(
                                                "Train Metric on %s: %s"
                                                % (
                                                    settings.YTRAIN_LABEL,
                                                    table.trainmetricname,
                                                )
                                            )
                                        with tags.div(_class="col-xs-6"):
                                            tags.b(
                                                "Test Metric on %s: %s"
                                                % (lbl, table.testmetricname)
                                            )
                                with tags.div(id=tabid, _class="AbedRanks"):
                                    pass
                                tags.hr()
                                write_table_json(table, tabid)
            tags.comment("./container")
        with tags.div(id="footer"):
            pass

    return str(doc)

def write_table_json(table, tabid):
    data = []
    averages = next((row for _id, row in table if _id == "Average"), None)
    headers = table.headers[1:]
    for hdr, avg in zip(headers, averages):
        data.append({"name": hdr, "time": float(avg)})
    fname = os.path.join(settings.OUTPUT_DIR, "html", tabid + ".json")
    with open(fname, "w") as fid:
        fid.write(json.dumps(data))
    info("Created output file: %s" % fname)

def write_tables_html(doc):
    fname = os.path.join(settings.OUTPUT_DIR, "html", AbedHTMLTypes.RANK_GRAPHS[-1])
    with open(fname, "w") as fid:
        fid.write(doc)
    info("Created output file: %s" % fname)


# -*- coding: utf-8 -*-
# @Time    : 2020/3/24 18:09
# @Author  : Sinclair
# @Email   : 1178211058@qq.com
# @File    : urls.py
from django.urls import path
from . import views

app_name = "blog"
urlpatterns = [
    path("detail/<int:pk>", views.detail, name="detail"),
    path("comment/<int:pk>", views.comment, name="comment"),
    path("add", views.add, name="add"),
]


from __future__ import annotations
from . import mp_out_utils
from ..cgt_naming import COLLECTIONS, cgt_defaults
from ..cgt_bpy import cgt_bpy_utils, cgt_collection, cgt_object_prop

class MPPoseOutputNode(mp_out_utils.BpyOutputNode):
    pose = []
    col_name = COLLECTIONS.pose
    parent_col = COLLECTIONS.drivers

    def __init__(self):
        data = cgt_defaults
        references = {}
        for k, v in data.pose.items():
            references[k] = v

        self.pose = cgt_bpy_utils.add_empties(references, 0.005, suffix='cgt_')
        for ob in self.pose:
            cgt_object_prop.set_custom_property(ob, "cgt_id", data.identifier)

        cgt_collection.add_list_to_collection(self.col_name, self.pose, self.parent_col)

    def update(self, data, frame):
        loc, rot, sca = data
        for data, method in zip([loc, rot, sca], [self.translate, self.euler_rotate, self.scale]):
            try:
                method(self.pose, data, frame)
            except IndexError:
                pass
        return data, frame


def check(candy):
    return len(set(candy)) == 1

def teacher(N, candy):
    for i in range(N):
        if candy[i] % 2 != 0:
            candy[i] += 1
    candy = [x // 2 for x in candy]
    half = candy[1:] + candy[:1]
    candy = list(map(lambda x, y: x + y, candy, half))
    return candy

def process():
    N, candy = int(input()), list(map(int, input().split()))
    candy = teacher(N, candy)
    cnt = 0
    while not check(candy):
        cnt += 1
        candy = teacher(N, candy)
    return cnt

T = int(input())
for _ in range(T):
    print(process())


"""Hetzner Cloud plugin for integration tests."""
from __future__ import absolute_import, print_function

from os.path import isfile

from lib.cloud import (
    CloudProvider,
    CloudEnvironment,
    CloudEnvironmentConfig,
)

from lib.util import ConfigParser, display

class HcloudCloudProvider(CloudProvider):
    """Hetzner Cloud provider plugin. Sets up cloud resources before
       delegation.
    """

    def __init__(self, args):
        """
        :type args: TestConfig
        """
        super(HcloudCloudProvider, self).__init__(args)

    def filter(self, targets, exclude):
        """Filter out the cloud tests when the necessary config and resources are not available.
        :type targets: tuple[TestTarget]
        :type exclude: list[str]
        """
        if isfile(self.config_static_path):
            return

        super(HcloudCloudProvider, self).filter(targets, exclude)

    def setup(self):
        """Setup the cloud resource before delegation and register a cleanup callback."""
        super(HcloudCloudProvider, self).setup()

        if isfile(self.config_static_path):
            self.config_path = self.config_static_path
            return True

        return False

class HcloudCloudEnvironment(CloudEnvironment):
    """Hetzner Cloud cloud environment plugin. Updates integration test environment
       after delegation.
    """

    def get_environment_config(self):
        parser = ConfigParser()
        parser.read(self.config_path)

        env_vars = dict(
            HCLOUD_TOKEN=parser.get('default', 'hcloud_api_token'),
        )

        ansible_vars = dict(
            hcloud_prefix=self.resource_prefix,
        )

        ansible_vars.update(dict((key.lower(), value) for key, value in env_vars.items()))

        return CloudEnvironmentConfig(
            env_vars=env_vars,
            ansible_vars=ansible_vars,
        )


# -*- coding: UTF-8 -*-

from sklearn.linear_model import LinearRegression

# from sklearn import linear_model

# 两个特征
X = [[5, 2], [2, 5], [3, 9]]
y = [0, 1, 2]

model = LinearRegression()
model.fit(X, y)

# 截距
print(model.intercept_)
# 斜率
print(model.coef_)

# 预测
y_pred = model.predict([[1, 1], [3, 3]])
print(y_pred)


import pytest

# # Create fixture
# @pytest.fixture
# def num_list():
#
#     # Create list
#     list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
#     return list

#Create method
def test_sum(num_list):
     # Initialize sum
     sum = 0

     # Add number in the list
     for n in num_list:
         sum += n

     # Assertion
     assert sum == 55


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 17 10:18:00 2021

@author: Mels
"""
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 10 15:03:46 2021

@author: Mels
"""
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import scipy.special as scpspc
import matplotlib as mpl
from matplotlib.patches import Rectangle
import tensorflow as tf

class Plot:
    def __init__(self):
        self.rdist_params=None
        self.rdist_var=None
        self.rdist_opt=None
        self.xdist_pararms=None
        self.xdist_var=None
        self.ydist_pararms=None
        self.ydist_var=None
        self.xydist_opt=None

    def model_summary(self):
        print('\n\n_________________________MODEL SUMMARY_________________________')
        print('- For the first channel, of the '+str(self.ch10.pos.shape[0])+' localizations, '
              +str(self.ch10.pos.shape[0]-self.ch1.pos.shape[0])+' have been filtered out! ('
              +str(round((1-(self.ch1.pos.shape[0]/self.ch10.pos.shape[0]))*100,1))+'%)')
        print('- For the second channel of the '+str(self.ch20.pos.shape[0])+' localizations, '
              +str(self.ch20.pos.shape[0]-self.ch2.pos.shape[0])+' have been filtered out! ('
              +str(round((1-(self.ch2.pos.shape[0]/self.ch20.pos.shape[0]))*100,1))+'%)')

        if self.rdist_params is not None:
            print('- The absolute distribution fit returned the values of [\u03BC,\u03C3]=['
                  +str(self.rdist_params[0])+', '+str(self.rdist_params[1])+'+/-'+str(self.rdist_var[0])+']nm.')
            if self.rdist_opt is not None:
                print('The optimal values were ['+str(self.rdist_opt[0])+', '+str(self.rdist_opt[1])+']nm.')
        if self.xdist_params is not None:
            print('- The relative distribution fit returned the values of [\u03BC\N{SUBSCRIPT ONE},\u03C3\N{SUBSCRIPT ONE}]=['
                  +str(self.xdist_params[0])+', '+str(self.xdist_params[1])+']+/-['
                  +str(self.xdist_var[0][0])+', '+str(self.xdist_var[1][1])+']nm,')
            print('[\u03BC\N{SUBSCRIPT TWO},\u03C3\N{SUBSCRIPT TWO}]=['
                  +str(self.ydist_params[0])+', '+str(self.ydist_params[1])+']+/-['
                  +str(self.ydist_var[0][0])+', '+str(self.ydist_var[1][1])+']nm.')
            if self.xydist_opt is not None:
                print('The optimal values were ['+str(self.xydist_opt[0])+', '+str(self.xydist_opt[1])+']nm.')
        print('\n\n')

    def ErrorDist(self, pos1, pos2):
    # Generates the error, average and radius
        dist = np.sqrt( np.sum( ( pos1 - pos2 )**2, axis = 1) )
        return dist, np.average(dist), np.sqrt(np.sum(pos1**2,1))

    #%% Plotting the error
    def ErrorPlot(self, nbins=30):
        ## Coupling Datasets if not done already
        if not self.linked: raise Exception('Dataset should first be linked before registration errors can be derived!')
        pos1_original=self.ch1.pos_all()
        pos2_original=self.ch20linked.pos_all()
        pos1=self.ch1.pos_all()
        pos2=self.ch2.pos_all()

        # Calculating the error
        dist1, avg1, r1 = self.ErrorDist(pos1, pos2)
        if self.ch20linked.pos_all() is not None:
            dist2, avg2, r2 = self.ErrorDist(pos1_original, pos2_original)

        ## Plotting
        if self.ch20linked.pos_all() is not None: fig, ((ax3, ax4), (ax1, ax2)) = plt.subplots(2,2)
        else: fig, (ax1, ax2) = plt.subplots(2)

        # plotting the histogram
        n1 = ax1.hist(dist1, label='Mapped', alpha=.8, edgecolor='red', color='tab:orange', bins=nbins)
        if self.ch20linked.pos_all() is not None:
            n1 = ax3.hist(dist1, label='Mapped', alpha=.8, edgecolor='red', color='tab:orange', bins=nbins)
            n2 = ax3.hist(dist2, label='Original', alpha=.8, edgecolor='red', color='tab:blue', bins=nbins)
        else:
            n2=[0]
        ymax = np.max([np.max(n1[0]), np.max(n2[0])])*1.1

        # plotting the FOV
        ax2.plot(r1, dist1, 'r.', alpha=.4, label='Mapped error')
        if self.ch20linked.pos_all() is not None:
            ax4.plot(r1, dist1, 'r.', alpha=.4, label='Mapped error')
            ax4.plot(r2, dist2, 'b.', alpha=.4, label='Original error')
        else:
            r2=np.array(0)
        xmax= np.max((np.max(r1),np.max(r2)))*1.1

        # Plotting the averages as vlines
        ax1.vlines(avg1, color='green', ymin=0, ymax=ymax, label=('avg mapped = '+str(round(avg1,2))))
        if self.ch20linked.pos_all() is not None:
            ax3.vlines(avg2, color='purple', ymin=0, ymax=ymax, label=('avg original = '+str(round(avg2,2))))
            ax3.vlines(avg1, color='green', ymin=0, ymax=ymax, label=('avg mapped = '+str(round(avg1,2))))

        # Plotting the averages as hlines
        ax2.hlines(avg1, color='green', xmin=0, xmax=xmax, label=('average mapped = '+str(round(avg1,2))))
        if self.ch20linked.pos_all() is not None:
            ax4.hlines(avg2, color='purple', xmin=0, xmax=xmax, label=('average original = '+str(round(avg2,2))))
            ax4.hlines(avg1, color='green', xmin=0, xmax=xmax, label=('average mapped = '+str(round(avg1,2))))

        # Some extra plotting parameters
        ax1.set_title('Zoomed in on Mapping Error')
        ax1.set_ylim([0,ymax])
        ax1.set_xlim(0)
        ax1.set_xlabel('Absolute error [nm]')
        ax1.set_ylabel('# of localizations')
        ax1.legend(loc='upper right')

        ax2.set_title('Zoomed in on Mapping Error')
        ax2.set_ylim(0)
        ax2.set_xlim([0,xmax])
        ax2.set_xlabel('FOV [nm]')
        ax2.set_ylabel('Absolute Error')
        ax2.legend(loc='upper right')

        if self.ch20linked.pos_all() is not None:
            ax3.set_title('Comparisson')
            ax3.set_ylim([0,ymax])
            ax3.set_xlim(0)
            ax3.set_xlabel('Absolute error [nm]')
            ax3.set_ylabel('# of localizations')
            ax3.legend(loc='upper right')

            ax4.set_title('Comparisson')
            ax4.set_ylim(0)
            ax4.set_xlim([0,xmax])
            ax4.set_xlabel('FOV [nm]')
            ax4.set_ylabel('Absolute Error')
            ax4.legend(loc='upper right')

        fig.tight_layout()
        fig.show()
        if self.ch20linked.pos_all() is not None:
            return avg1, avg2, fig, (ax3, ax1, ax4, ax2)
        else:
            return avg1, fig, (ax1, ax2)

    def ErrorDistribution(self, nbins=30):
    # just plots the error distribution after mapping
        if not self.linked: raise Exception('Dataset should first be linked before registration errors can be derived!')
        pos1=self.ch1.pos_all()
        pos2=self.ch2.pos_all()
        dist1, avg1, r1 = self.ErrorDist(pos1, pos2)

        plt.figure()
        n1 = plt.hist(dist1, label=('Mapped Error = '+str(round(avg1,2))+'nm'), alpha=.8, edgecolor='red', color='tab:orange', bins=nbins)
        ymax = np.max(n1[0]*1.1)
        #plt.title('Zoomed in on Mapping Error')
        plt.ylim([0,ymax])
        plt.xlim(0)
        plt.xlabel('error [nm]')
        plt.ylabel('# of localizations')
        plt.legend(loc='upper right')
        plt.tight_layout()

    def ErrorDistribution_xy(self, nbins=30, xlim=31, error=None, mu=None, fit_data=True, clusters=False):
        if not self.linked: raise Exception('Dataset should first be linked before registration errors can be derived!')
        if mu is None: mu=0
        if clusters:
            pos1=self.ch1.ClusterCOM()[0]
            pos2=self.ch2.ClusterCOM()[0]
            pos1, pos2=self.kNearestNeighbour(pos1, pos2, k=-1, maxDistance=5000)
            pos1=pos1.numpy()
            pos2=pos2.numpy()
        else:
            pos1=self.ch1.pos_all()
            pos2=self.ch2.pos_all()

        fig, ax = plt.subplots(1,2,figsize=(12,6))
        distx=pos1[:,0]-pos2[:,0]
        disty=pos1[:,1]-pos2[:,1]
        mask=np.where(distx<xlim,True, False)*np.where(disty<xlim,True,False)
        distx=distx[mask]
        disty=disty[mask]
        nx = ax[0].hist(distx, range=[-xlim,xlim], label='N='+str(pos1.shape[0]),alpha=.8,
                        edgecolor='red', color='tab:orange', bins=nbins)
        ny = ax[1].hist(disty, range=[-xlim,xlim], label='N='+str(pos1.shape[0]),alpha=.8,
                        edgecolor='red', color='tab:orange', bins=nbins)

        if fit_data: ## fit bar plot data using curve_fit
            def func(r, mu, sigma):
                return np.exp(-(r - mu) ** 2 / (2 * sigma ** 2)) / (np.sqrt(2*np.pi)*sigma)

            Nx = pos1.shape[0] * ( nx[1][1]-nx[1][0] )
            Ny = pos1.shape[0] * ( ny[1][1]-ny[1][0] )
            xn=(nx[1][:-1]+nx[1][1:])/2
            yn=(ny[1][:-1]+ny[1][1:])/2
            poptx, pcovx = curve_fit(func, xn, nx[0]/Nx, p0=[np.average(distx), np.std(distx)])
            popty, pcovy = curve_fit(func, yn, ny[0]/Ny, p0=[np.average(disty), np.std(disty)])
            x = np.linspace(-xlim, xlim, 1000)
            yx = func(x, *poptx)*Nx
            yy = func(x, *popty)*Ny
            ax[0].plot(x, yx, c='g',label=(r'fit: $\mu$='+str(round(poptx[0],2))+', $\sigma$='+str(round(poptx[1],2))+'nm'))
            ax[1].plot(x, yy, c='g',label=(r'fit: $\mu$='+str(round(popty[0],2))+', $\sigma$='+str(round(popty[1],2))+'nm'))
            ymax = np.max([np.max(nx[0]),np.max(ny[0]), np.max(yx), np.max(yy)])*1.1

            ## plot how function should look like
            if error is not None:
                sgm=error+mu
                opt_yx = func(x, 0, sgm)*Nx
                opt_yy = func(x, 0, sgm)*Ny
                ax[0].plot(x, opt_yx, c='b',label=(r'optimum: $\mu$='+str(round(mu,2))+', $\sigma$='+str(round(sgm,2))+'nm'))
                ax[1].plot(x, opt_yy, c='b',label=(r'optimum: $\mu$='+str(round(mu,2))+', $\sigma$='+str(round(sgm,2))+'nm'))
                if np.max([np.max(opt_yx),np.max(opt_yy)])>ymax: ymax=np.max([np.max(opt_yx),np.max(opt_yy)])*1.1
        else: ymax=np.max([np.max(nx[0]),np.max(ny[0])])*1.1

        ax[0].set_ylim([0,ymax])
        ax[0].set_xlim(-xlim,xlim)
        ax[0].set_xlabel('x-error [nm]')
        ax[0].set_ylabel('# of localizations')
        ax[0].legend(loc='upper right')

        ax[1].set_ylim([0,ymax])
        ax[1].set_xlim(-xlim,xlim)
        ax[1].set_xlabel('y-error [nm]')
        ax[1].set_ylabel('# of localizations')
        ax[1].legend(loc='upper right')
        fig.tight_layout()

        if fit_data:
            self.xdist_params=np.array([np.round(poptx[0],2),np.round(poptx[1],2)])
            self.xdist_var=np.array([np.round(pcovx[0],2),np.round(pcovx[1],2)])
            self.ydist_params=np.array([np.round(popty[0],2),np.round(popty[1],2)])
            self.ydist_var=np.array([np.round(pcovy[0],2),np.round(pcovy[1],2)])
            if error is not None: self.xydist_opt=np.array([np.round(mu,2),np.round(sgm,2)])
            else: self.xydist_opt=None
            return poptx, popty, pcovx, pcovy
        else: return None, None

    def ErrorDistribution_r(self, nbins=30, xlim=31, error=None, mu=None, fit_data=True, plot_on=True, clusters=False):
        if not self.linked: raise Exception('Dataset should first be linked before registration errors can be derived!')
        if mu is None: mu=0
        if clusters:
            pos1=self.ch1.ClusterCOM()[0]
            pos2=self.ch2.ClusterCOM()[0]
            pos1, pos2=self.kNearestNeighbour(pos1, pos2, k=-1, maxDistance=5000)
            pos1=pos1.numpy()
            pos2=pos2.numpy()
        else:
            pos1=self.ch1.pos_all()
            pos2=self.ch2.pos_all()

        # Calculating the error
        dist, avg, r = self.ErrorDist(pos1, pos2)
        dist=dist[np.argwhere(dist<xlim)]

        # plotting the histogram
        if plot_on: plt.figure()
        if plot_on: n=plt.hist(dist, range=[0,xlim], label='N='+str(pos1.shape[0]), alpha=.8, edgecolor='red', color='tab:orange', bins=nbins)
        else: n=np.histogram(dist, range=[0,xlim], bins=nbins)
        #plt.axvline(x=avg, label='average='+str(round(avg,2))+'[nm]')
        ymax = np.max(n[0])*1.1

        if fit_data: ## fit bar plot data using curve_fit
            def func(r, sigma, mu):
                # from Churchman et al 2006
                sigma2=sigma**2
                return (r/sigma2)*np.exp(-(mu**2+r**2)/2/sigma2)*scpspc.jv(0, r*mu/sigma2)

            N = pos1.shape[0] * ( n[1][1]-n[1][0] )
            xn=(n[1][:-1]+n[1][1:])/2
            popt, pcov = curve_fit(func, xn, n[0]/N, p0=[np.std(xn), np.average(xn)])
            x = np.linspace(0, xlim, 1000)
            y = func(x, *popt)*N
            if plot_on: plt.plot(x, y, c='g',label=(r'fit: $\mu$='+str(round(popt[1],2))+', $\sigma$='+str(round(popt[0],2))+'nm'))

            ## plot how function should look like
            if error is not None:
                sgm=error
                y = func(x, sgm, mu)*N
                if plot_on: plt.plot(x, y, c='b',label=(r'optimum: $\mu$='+str(round(mu,2))+', $\sigma$='+str(round(sgm,2))+'nm'))
                if np.max(y)>ymax: ymax=np.max(y)*1.1

        if plot_on: # Some extra plotting parameters
            plt.ylim([0,ymax])
            plt.xlim([0,xlim])
            plt.xlabel('Absolute error [nm]')
            plt.ylabel('# of localizations')
            plt.legend(loc='upper right')
            plt.tight_layout()

        if fit_data:
            self.rdist_params=np.array([np.round(popt[1],2),np.round(popt[0],2)])
            self.rdist_var=np.round(pcov[0],2)
            if error is not None: self.rdist_opt=np.array([np.round(mu,2),np.round(sgm,2)])
            else: self.rdist_opt=None
            return popt, pcov
        else: return None

    #%% plotting the error in a [x1, x2] plot like in the paper
    def ErrorFOV(self, other=None, maxDistance=30, ps=1, cmap='seismic', figsize=None, title=None, precision=750,
                 placement='right', colorbar=True, center=[3,3], clusters=False, text=True, alpha=.8, norm=1):
        def prepdata(pos, z, precision=750):
            min1=np.min(pos[:,0])/1000
            min2=np.min(pos[:,1])/1000
            max1=np.max(pos[:,0])/1000
            max2=np.max(pos[:,1])/1000
            density=pos.shape[0]/((max1-min1)*(max2-min2))*precision/1000
            pos[:,0]-=min1
            pos[:,1]-=min2

            N1=int(np.max(pos[:,0])/precision)+1
            N2=int(np.max(pos[:,1])/precision)+1
            Z=np.zeros([N2,N1], dtype=float)
            #N=np.zeros([N2,N1], dtype=float)
            for n in range(z.shape[0]): #calculate average displacement per cell
                i=np.floor(pos[n,1]/precision).astype('int')
                j=np.floor(pos[n,0]/precision).astype('int')
                Z[i,j]+=z[n]/density
            #    N[i,j]+=1
            #for i in range(N2):
            #    for j in range(N1):
            #        if N[i,j]>1.: Z[i,j]/=N[i,j]

            X=np.linspace(min1,max1, N1)
            Y=np.linspace(min2,max2, N2)
            return X,Y,Z

        if clusters:
            pos1=self.ch1.ClusterCOM()[0]
            pos2=self.ch2.ClusterCOM()[0]
            pos1, pos2=self.kNearestNeighbour(pos1, pos2, k=-1, maxDistance=5000)
            pos1=pos1.numpy()
            pos2=pos2.numpy()
        else:
            pos1=self.ch1.pos.numpy()
            pos2=self.ch2.pos.numpy()
        dist = (pos1-pos2)/norm

        if not self.linked and not clusters: raise Exception('Dataset should first be linked before registration errors can be derived!')
        if dist.shape==(0,): raise ValueError('No neighbours found for channel 1')

        if figsize is None: figsize=(14,6)
        fig, ax = plt.subplots(1,2, figsize=figsize,sharex = False,sharey=False,constrained_layout=True)
        xlim=(tf.reduce_min(pos1[:,0]/1000),tf.reduce_max(pos1[:,0]/1000))
        ylim=(tf.reduce_min(pos1[:,1]/1000),tf.reduce_max(pos1[:,1]/1000))

        X1,Y1,Z1=prepdata(pos1, dist[:,0], precision=precision)
        X2,Y2,Z2=prepdata(pos1, dist[:,1], precision=precision)
        vmin=np.min([np.min(Z1),np.min(Z2)])
        vmax=np.max([np.max(Z1),np.max(Z2)])
        if vmin<0:
            vm=np.max([-vmin, vmax])
            norm=mpl.colors.Normalize(vmin=-vm, vmax=vm, clip=False)
        else:
            norm=mpl.colors.Normalize(vmin=0, vmax=vmax, clip=False)

        ax[1].contourf(X2,Y2,Z2, norm=norm, cmap=cmap, alpha=.8)
        ax[0].contourf(X1,Y1,Z1, norm=norm, cmap=cmap, alpha=.8)

        ax[0].set_xticks([])
        ax[0].set_yticks([])
        ax[0].set_xlim(xlim)
        ax[0].set_ylim(ylim)
        ax[0].set_title('x-error')
        ax[0].set_aspect('equal', 'box')

        #ax[0].scatter(pos1[:,0]/1000, pos1[:,1]/1000, s=ps, c=dist[:,0],
        #              cmap=cmap, norm=norm, alpha=.8, lw=0)
        #ax[1].scatter(pos2[:,0]/1000, pos2[:,1]/1000,  s=ps, c=dist[:,1],
        #              cmap=cmap, norm=norm, alpha=.8, lw=0)

        ax[1].set_xticks([])
        ax[1].set_yticks([])
        ax[1].set_xlim(xlim)
        ax[1].set_ylim(ylim)
        ax[1].set_title('y-error')
        ax[1].set_aspect('equal', 'box')

        if placement=='bottom':
            shrink=0.8
            aspect=20
        else:
            shrink=.8
            aspect=40
        if colorbar:
            cb=fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap),
                                     shrink=shrink, aspect=aspect)#, ticks=[-1000,0,1000])
            #cb.ax.set_yticklabels(['10', '0', '10'])
        if title is not None: fig.suptitle(title)
        return fig,ax

    def ErrorFOVr(self, fig, ax1, ax2, ps=7, cmap='seismic', placement='right', colorbar=True, clusters=False):
        if clusters:
            pos1=self.ch1.ClusterCOM()[0]
            pos2=self.ch2.ClusterCOM()[0]
            pos1, pos2=self.kNearestNeighbour(pos1, pos2, k=-1, maxDistance=5000)
            pos1=pos1.numpy()
            pos2=pos2.numpy()
        else:
            pos1=self.ch1.pos.numpy()
            pos2=self.ch2.pos.numpy()
        dist = tf.sqrt(tf.reduce_sum(tf.square(pos1-pos2),axis=1))

        if not self.linked and not clusters: raise Exception('Dataset should first be linked before registration errors can be derived!')
        if dist.shape==(0,): raise ValueError('No neighbours found for channel 1')

        xlim=(tf.reduce_min(pos1[:,0]/1000),tf.reduce_max(pos1[:,0]/1000))
        ylim=(tf.reduce_min(pos1[:,1]/1000),tf.reduce_max(pos1[:,1]/1000))

        vmax=np.max(dist)
        norm=mpl.colors.Normalize(vmin=0, vmax=vmax, clip=False)

        ax1.scatter(pos1[:,0]/1000, pos1[:,1]/1000, s=ps,
                      cmap=cmap, norm=norm, alpha=.8, lw=0)
        ax1.set_xticks([])
        ax1.set_yticks([])
        ax1.set_xlim(xlim)
        ax1.set_ylim(ylim)
        ax1.set_aspect('equal', 'box')

        ax2.scatter(pos2[:,0]/1000, pos2[:,1]/1000, s=ps, c=dist,
                      cmap=cmap, norm=norm, alpha=.8, lw=0)
        ax2.set_xticks([])
        ax2.set_yticks([])
        ax2.set_xlim(xlim)
        ax2.set_ylim(ylim)
        ax2.set_aspect('equal', 'box')

        fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap))
        #fig.tight_layout()
        return fig,ax1, ax2

    #%% Channel to matrix fn
    def isin_domain(self, pos):
    # checks if pos is within bounds
        return ( pos[0] > self.bounds[0,0] and pos[1] > self.bounds[1,0] and
                pos[0] < self.bounds[0,1] and pos[1] < self.bounds[1,1] )

    def generate_channel(self, precision=10, heatmap=False, bounds=None):
    # Generates the channels as matrix
        print('Generating Channels as matrix...')
        self.precision=precision
        locs1 = self.ch1.pos_all()  / precision
        locs2 = self.ch2.pos_all()  / precision

        self.bounds = np.empty([2,2], dtype = float)
        if bounds is None:
            self.bounds[0,0] = np.min([ np.min(locs1[:,0]), np.min(locs2[:,0])])
            self.bounds[0,1] = np.max([ np.max(locs1[:,0]), np.max(locs2[:,0])])
            self.bounds[1,0] = np.min([ np.min(locs1[:,1]), np.min(locs2[:,1])])
            self.bounds[1,1] = np.max([ np.max(locs1[:,1]), np.max(locs2[:,1])])
        else:
            self.bounds=bounds / self.precision
        self.size_img = np.abs(np.round( (self.bounds[:,1] - self.bounds[:,0]) , 0).astype('int')    )
        self.axis = np.array([ self.bounds[1,:], self.bounds[0,:]]) * self.precision
        self.axis = np.reshape(self.axis, [1,4])[0]

        # generating the matrices to be plotted
        self.channel1 = self.generate_matrix(locs1, heatmap)
        self.channel2 = self.generate_matrix(locs2, heatmap)

    def generate_matrix(self, locs, heatmap=False):
    # takes the localizations and puts them in a channel
        channel = np.zeros([self.size_img[0]+1, self.size_img[1]+1], dtype = int)
        for i in range(locs.shape[0]):
            loc = np.round(locs[i,:],0).astype('int')
            if self.isin_domain(loc):
                loc -= np.round(self.bounds[:,0],0).astype('int') # place the zero point on the left
                if heatmap: channel[loc[0], loc[1]] += 1
                else: channel[loc[0], loc[1]] = 1
        return channel

    #%% Plotting Channels
    def show_channel(self, pos, color='red', ps=3, alpha=1, fig=None,  ax=None,
                     figsize=(3,6), addpatch=True, lims=None):
        print('Plotting...')
        if figsize is None: figsize=(4,int(self.imgshape[0]/self.imgshape[1])*4)
        if fig is None: fig=plt.figure(figsize=figsize)
        if ax is None: ax=fig.add_subplot(111)
        ax.scatter(pos[:,0]/1000,pos[:,1]/1000, color=color, marker='.', s=ps, alpha=alpha, lw=0)
        ax.set_xticks([])
        ax.set_yticks([])
        if lims is None:
            ax.set_xlim([np.min(self.ch1.pos[:,0])/1000,np.max(self.ch1.pos[:,0])/1000])
            ax.set_ylim([np.min(self.ch1.pos[:,1])/1000,np.max(self.ch1.pos[:,1])/1000])
        else:
            ax.set_xlim(lims[0,:]/1000)
            ax.set_ylim(lims[1,:]/1000)
        fig.tight_layout()

        if addpatch:
            x1=np.min(self.ch1.pos[:,0])/1000 +3
            x2=np.min(self.ch1.pos[:,1])/1000 +3
            ax.add_patch(Rectangle((x1,x2), 10, .5, ec='black', fc='black'))
            ax.text(x1, x2+.5, r'10$\mu$m', ha='left', va='bottom')
        return fig, ax

    def plot_channel(self, colormap='viridis'):
        print('Plotting...')
        label=['y-position [\u03bcm]', 'x-position [\u03bcm]']

        # plotting all channels
        plt.figure()
        if self.ch20.pos_all() is not None: plt.subplot(131)
        else: plt.subplot(121)
        plt.imshow(self.channel1, extent = self.axis/1000, cmap=colormap)
        plt.xlabel(label[0])
        plt.ylabel(label[1])
        plt.title('original channel 1')
        plt.tight_layout()

        if self.ch20.pos_all() is not None: plt.subplot(132)
        else: plt.subplot(122)
        plt.imshow(self.channel2, extent = self.axis/1000, cmap=colormap)
        plt.xlabel(label[0])
        plt.ylabel(label[1])
        plt.title('mapped channel 2')
        plt.tight_layout()

        if self.ch20.pos_all() is not None:
            plt.subplot(133)
            plt.imshow(self.channel2_original, extent = self.axis/1000, cmap=colormap)
            plt.xlabel(label[0])
            plt.ylabel(label[1])
            plt.title('original channel 2')
            plt.tight_layout()

    def plot_1channel(self, channel1=None, figsize=None, title=None, colormap='viridis'):
        if channel1 is None: channel1=self.channel1
        # plotting all channels
        if figsize is None: fig=plt.figure()
        else:  fig=plt.figure(figsize=figsize)
        ax = fig.add_subplot(111)

        ax.imshow(channel1, extent = self.axis/1000, cmap=colormap)
        ax.set_xlabel('x-position [\u03bcm]')
        ax.set_ylabel('y-position [\u03bcm]')
        if title is None: ax.set_title('Single Channel view')
        else: ax.set_title(title)
        fig.tight_layout()
        return fig, ax

    #%% Plotting the Grid
    def PlotSplineGrid(self, ch1=None, ch2=None, ch20=None, fig=None, ax=None, figsize=None,
                       locs_markersize=25, CP_markersize=20, d_grid=.1, Ngrids=4, lw=1,
                       plotmap=False, plotpoints=False, plotCP=False):
        '''
        Plots the grid and the shape of the grid in between the Control Points

        Parameters
        ----------
        ch1 , ch2 , ch20 : Nx2 tf.float32 tensor
            The tensor containing the localizations.
        d_grid : float, optional
            The precission of the grid we want to plot in between the
            ControlPoints. The default is .1.
        lines_per_CP : int, optional
            The number of lines we want to plot in between the grids.
            Works best if even. The default is 1.
        locs_markersize : float, optional
            The size of the markers of the localizations. The default is 10.
        CP_markersize : float, optional
            The size of the markers of the Controlpoints. The default is 8.

        Returns
        -------
        None.

        '''
        print('Plotting the Spline Grid...')
        if ch1 is None:
            ch1=self.ch1.pos
            ch2=self.ch2.pos
            ch20=self.ch20.pos

        ## The original points
        def zero_axis(pts):
                return (tf.Variable( tf.stack([
                pts[:,0] - (self.x1_min-1 - self.edge_grids) * self.gridsize,
                pts[:,1] - (self.x2_min-1 - self.edge_grids) * self.gridsize
                ], axis=-1), dtype=tf.float32, trainable=False))
        ch1=zero_axis(ch1)
        ch2=zero_axis(ch2)
        ch20=zero_axis(ch20)

        # plotting the localizations
        if fig is None:
            if figsize is None: fig=plt.figure()
            else:  fig=plt.figure(figsize=figsize)
        if ax is None: ax=fig.add_subplot(111)

        if plotpoints:
            ax.scatter(ch20[:,0],ch20[:,1], c='green', marker='.', s=locs_markersize, label='Original')
            ax.scatter(ch1[:,0],ch1[:,1], c='red', marker='.', s=locs_markersize, label='Target')
        if plotmap:
            ax.scatter(ch2[:,0],ch2[:,1], c='blue', marker='.', s=locs_markersize, label='Mapped')
        if plotCP:
            ax.scatter(self.ControlPoints[:,:,0]*self.gridsize, self.ControlPoints[:,:,1]*self.gridsize,
                        c='b', marker='d', s=CP_markersize, label='ControlPoints')

        ## Horizontal Grid
        x1_grid = tf.range(0, tf.reduce_max(self.ControlPoints[:,:,0])+d_grid, delta=d_grid, dtype=tf.float32)
        x2_grid = tf.range(0, tf.reduce_max(self.ControlPoints[:,:,1])+d_grid, delta=1/Ngrids, dtype=tf.float32)
        Grid = tf.reshape(tf.stack(tf.meshgrid(x1_grid, x2_grid), axis=-1) , (-1,2))
        if self.SplinesModel is not None: Grid = self.SplinesModel( Grid ) * self.gridsize
        else: Grid = Grid*self.gridsize
        (nn, i,j)=(x1_grid.shape[0],0,0)
        while i<Grid.shape[0]:
            if j%Ngrids==0:
                ax.plot(Grid[i:i+nn,0], Grid[i:i+nn,1], c='b', lw=lw)
            else:
                ax.plot(Grid[i:i+nn,0], Grid[i:i+nn,1], c='c', lw=lw)
            i+=nn
            j+=1

        ## Vertical Grid
        x1_grid = tf.range(0, tf.reduce_max(self.ControlPoints[:,:,0])+d_grid, delta=1/Ngrids, dtype=tf.float32)
        x2_grid = tf.range(0, tf.reduce_max(self.ControlPoints[:,:,1])+d_grid, delta=d_grid, dtype=tf.float32)
        Grid = tf.gather(tf.reshape(tf.stack(tf.meshgrid(x2_grid, x1_grid), axis=-1) , (-1,2)), [1,0], axis=1)
        if self.SplinesModel is not None: Grid = self.SplinesModel( Grid ) * self.gridsize
        else: Grid = Grid*self.gridsize
        (nn, i,j)=(x2_grid.shape[0],0,0)
        while i<Grid.shape[0]:
            if j%Ngrids==0:
                ax.plot(Grid[i:i+nn,0], Grid[i:i+nn,1], c='b', lw=lw)
            else:
                ax.plot(Grid[i:i+nn,0], Grid[i:i+nn,1], c='c', lw=lw)
            i+=nn
            j+=1

        ax.legend(loc='upper right')
        fig.tight_layout()
        return fig, ax


import numpy as np
import netCDF4 as nc
import csv

import os
import fnmatch

def recursive_glob(rootdir = '.', pattern = '*'):
    """Search recursively for files matching a specified pattern."""

    matches = []
    for root, dirnames, filenames in os.walk(rootdir):
        for filename in fnmatch.filter(filenames, pattern):
            matches.append(os.path.join(root, filename))

    return matches

import read_host_info
sv = read_host_info.read_host_info()
bathy_dir = sv['in_dir']

# first, glob all data path
bathy_raw_dir = bathy_dir + 'bathy/raw/'
flist = recursive_glob(rootdir=bathy_raw_dir, pattern='*.xyz')

# read csv files
lat = []
lon = []
z = []
for fn in flist:
    f = open(fn, 'rb')
    rd = csv.reader(f, delimiter='\t')
    next(rd, None)
    for row in rd:
        # stn = row[0]
        # if stn=='H10374':
        #     print(fn)
        lon.append(float(row[1]))
        lat.append(float(row[2]))
        z.append(float(row[3]))

lon = np.array(lon)
lat = np.array(lat)
z = np.array(z)

# special treatment for Adams Inlet
lat_ai = []
lon_ai = []
z_ai = []
f = open(bathy_dir+'bathy/adams_inlet/surveys.xyz', 'rb')
rd = csv.reader(f, delimiter='\t')
next(rd, None)
for row in rd:
    lon_ai.append(float(row[1]))
    lat_ai.append(float(row[2]))
    z_ai.append(float(row[3]))

lon_ai = np.array(lon_ai)
lat_ai = np.array(lat_ai)
z_ai = np.array(z_ai)

lon1 = -136.
lon2 = -135.75
msk = (lon_ai>lon1) & (lon_ai<lon2)
lon_ai = lon_ai[msk]
lat_ai = lat_ai[msk]
z_ai = z_ai[msk]

lon = np.concatenate((lon, lon_ai), axis=0)
lat = np.concatenate((lat, lat_ai), axis=0)
z = np.concatenate((z, z_ai))

fh = nc.Dataset(bathy_dir+'bathy_noaa.nc', 'w')
fh.createDimension('ct')
fh.createVariable('lon', 'd', ('ct'))
fh.createVariable('lat', 'd', ('ct'))
fh.createVariable('z', 'd', ('ct'))
fh.variables['lon'][:] = lon
fh.variables['lat'][:] = lat
fh.variables['z'][:] = z
fh.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for the transmart_loader module.
"""
import csv
from datetime import date, datetime
from os import path
from typing import List

import pytest

from transmart_loader.copy_writer import TransmartCopyWriter
from transmart_loader.transmart import DataCollection, Concept, Study, \
    TrialVisit, Visit, TreeNode, Patient, Observation, ValueType, StudyNode, \
    ConceptNode, CategoricalValue, Modifier, ObservationMetadata, \
    TextValue, DateValue, Dimension, DimensionType, RelationType, Relation, \
    TreeNodeMetadata, StudyMetadata

@pytest.fixture
def empty_collection() -> DataCollection:
    concepts: List[Concept] = []
    modifiers: List[Modifier] = []
    dimensions: List[Dimension] = []
    studies: List[Study] = []
    trial_visits: List[TrialVisit] = []
    patients: List[Patient] = []
    visits: List[Visit] = []
    ontology: List[TreeNode] = []
    observations: List[Observation] = []
    collection = DataCollection(concepts, modifiers, dimensions, studies,
                                trial_visits, visits, ontology, patients,
                                observations)
    return collection

@pytest.fixture
def simple_collection() -> DataCollection:
    concepts: List[Concept] = [
        Concept('dummy_code', 'Dummy variable', '\\dummy\\path',
                ValueType.Categorical),
        Concept('diagnosis_date', 'Diagnosis date', '\\diagnosis_date',
                ValueType.Date),
        Concept('extra_c1', 'Extra c1', '\\c1', ValueType.Categorical),
        Concept('extra_c2', 'Extra c2', '\\c1', ValueType.Categorical)]
    modifiers: List[Modifier] = [
        Modifier('missing_value', 'Missing value', '\\missing_value',
                 ValueType.Text),
        Modifier('sample_id', 'Sample ID', '\\sample_id',
                 ValueType.Numeric)]
    dimensions: List[Dimension] = [
        Dimension('sample', modifiers[1], DimensionType.Subject, 1)
    ]
    study_metadata = StudyMetadata(**{'conceptCodeToVariableMetadata': {
        'test_concept': {
            'name': 'variable_1',
            'type': 'DATETIME'
        }
    }})
    studies: List[Study] = [Study('test', 'Test study', study_metadata)]
    trial_visits: List[TrialVisit] = [
        TrialVisit(studies[0], 'Week 1', 'Week', 1)]
    patients: List[Patient] = [Patient('SUBJ0', 'male', [])]
    visits: List[Visit] = [
        Visit(patients[0], 'visit1', None, None, None, None, None, None, [])]
    top_node = StudyNode(studies[0])
    top_node.metadata = TreeNodeMetadata(
        {'Upload date': '2019-07-01'})
    top_node.add_child(ConceptNode(concepts[0]))
    top_node.add_child(ConceptNode(concepts[1]))

    node2 = TreeNode('Extra node')
    node2.add_child((ConceptNode(concepts[2])))
    node3 = TreeNode('Extra node')
    node3.add_child(ConceptNode(concepts[3]))

    ontology: List[TreeNode] = [top_node, node2, node3]
    observations: List[Observation] = [
        Observation(patients[0], concepts[0], visits[0], trial_visits[0],
                    date(2019, 3, 28), None, CategoricalValue('value')),
        Observation(patients[0], concepts[0], visits[0], trial_visits[0],
                    datetime(2019, 6, 26, 12, 34, 00),
                    datetime(2019, 6, 28, 16, 46, 13, 345),
                    CategoricalValue(None),
                    ObservationMetadata({
                        modifiers[0]: TextValue('Invalid')
                    })),
        Observation(patients[0], concepts[1], visits[0], trial_visits[0],
                    datetime(2019, 6, 26, 13, 50, 10), None,
                    DateValue(datetime(2018, 4, 30, 17, 10, 00)))
    ]
    collection = DataCollection(concepts, modifiers, dimensions, studies,
                                trial_visits, visits, ontology, patients,
                                observations)
    return collection

@pytest.fixture
def collection_with_relations() -> DataCollection:
    concepts: List[Concept] = [
        Concept('dummy_code', 'Dummy variable', '\\dummy\\path',
                ValueType.Categorical)]
    studies: List[Study] = [Study('test', 'Test study')]
    trial_visits: List[TrialVisit] = [
        TrialVisit(studies[0], 'Week 1', 'Week', 1)]
    patients: List[Patient] = [
        Patient('SUBJ0', 'male', []),
        Patient('SUBJ1', 'female', []),
        Patient('SUBJ2', 'female', [])
    ]
    visits: List[Visit] = [
        Visit(patients[0], 'visit1', None, None, None, None, None, None, [])]
    top_node = StudyNode(studies[0])
    top_node.add_child(ConceptNode(concepts[0]))
    ontology: List[TreeNode] = [top_node]
    relation_types = [RelationType('parent', None, None, None),
                      RelationType('sibling', 'Sibling of', True, True)]
    relations = [
        Relation(patients[0], relation_types[0], patients[1], None, None),
        Relation(patients[0], relation_types[0], patients[2], None, None),
        Relation(patients[1], relation_types[1], patients[2], True, True)]
    collection = DataCollection(concepts, [], [], studies,
                                trial_visits, visits, ontology, patients,
                                [], relation_types, relations)
    return collection

def get_column_values(file_path: str, column_name: str) -> List[str]:
    rows = []
    with open(file_path) as file:
        reader = csv.DictReader(file, delimiter="\t")
        for row in reader:
            rows.append(row[column_name])
    return rows

def test_load_empty_collection(tmp_path, empty_collection):
    target_path = tmp_path.as_posix()
    writer = TransmartCopyWriter(target_path)
    writer.write_collection(empty_collection)
    assert path.exists(target_path + '/i2b2metadata/i2b2_secure.tsv')
    assert path.exists(target_path + '/i2b2metadata/i2b2_tags.tsv')
    assert path.exists(target_path + '/i2b2demodata/concept_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/modifier_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/patient_mapping.tsv')
    assert path.exists(target_path + '/i2b2demodata/patient_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/encounter_mapping.tsv')
    assert path.exists(target_path + '/i2b2demodata/visit_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/study.tsv')
    assert path.exists(target_path + '/i2b2metadata/dimension_description.tsv')
    assert path.exists(
        target_path + '/i2b2metadata/study_dimension_descriptions.tsv')
    assert path.exists(target_path + '/i2b2demodata/trial_visit_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/observation_fact.tsv')

def test_load_simple_collection(tmp_path, simple_collection):
    target_path = tmp_path.as_posix()
    writer = TransmartCopyWriter(target_path)
    writer.write_collection(simple_collection)
    del writer

    assert path.exists(target_path + '/i2b2metadata/i2b2_secure.tsv')
    tree_node_paths = get_column_values(target_path + '/i2b2metadata/i2b2_secure.tsv', 'c_fullname')
    assert len(tree_node_paths) == 6
    assert all(elem in tree_node_paths for elem in [
        '\\Test study\\', '\\Test study\\Dummy variable\\', '\\Test study\\Diagnosis date\\',
        '\\Extra node\\', '\\Extra node\\Extra c1\\', '\\Extra node\\Extra c2\\'])
    assert path.exists(target_path + '/i2b2metadata/i2b2_tags.tsv')
    assert path.exists(target_path + '/i2b2demodata/concept_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/modifier_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/patient_mapping.tsv')
    assert path.exists(target_path + '/i2b2demodata/patient_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/encounter_mapping.tsv')
    assert path.exists(target_path + '/i2b2demodata/visit_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/study.tsv')
    assert path.exists(target_path + '/i2b2metadata/dimension_description.tsv')
    assert path.exists(
        target_path + '/i2b2metadata/study_dimension_descriptions.tsv')
    assert path.exists(target_path + '/i2b2demodata/trial_visit_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/observation_fact.tsv')

def test_load_collection_with_relations(tmp_path, collection_with_relations):
    target_path = tmp_path.as_posix()
    writer = TransmartCopyWriter(target_path)
    writer.write_collection(collection_with_relations)
    assert path.exists(target_path + '/i2b2metadata/i2b2_secure.tsv')
    assert path.exists(target_path + '/i2b2metadata/i2b2_tags.tsv')
    assert path.exists(target_path + '/i2b2demodata/concept_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/modifier_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/patient_mapping.tsv')
    assert path.exists(target_path + '/i2b2demodata/patient_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/encounter_mapping.tsv')
    assert path.exists(target_path + '/i2b2demodata/visit_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/study.tsv')
    assert path.exists(target_path + '/i2b2metadata/dimension_description.tsv')
    assert path.exists(
        target_path + '/i2b2metadata/study_dimension_descriptions.tsv')
    assert path.exists(target_path + '/i2b2demodata/trial_visit_dimension.tsv')
    assert path.exists(target_path + '/i2b2demodata/observation_fact.tsv')
    assert path.exists(target_path + '/i2b2demodata/relation_types.tsv')
    assert path.exists(target_path + '/i2b2demodata/relations.tsv')


#! /usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import math
import copy
import json
import shutil
import tarfile
import tempfile
import logging

# paddle
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from paddle.nn import LayerNorm as BertLayerNorm

# utils
from utils.bert_utils import cached_path

logger = logging.getLogger(__name__)

PRETRAINED_MODEL_ARCHIVE_MAP = {
    "bert-base-uncased": "https://bert-models.bj.bcebos.com/uncased_L-12_H-768_A-12.tar.gz",
    "bert-large-uncased": "https://bert-models.bj.bcebos.com/uncased_L-24_H-1024_A-16.tar.gz",
    "bert-base-cased": "https://bert-models.bj.bcebos.com/cased_L-12_H-768_A-12.tar.gz",
    "bert-large-cased": "https://bert-models.bj.bcebos.com/cased_L-24_H-1024_A-16.tar.gz",
    "bert-base-multilingual-uncased": "https://bert-models.bj.bcebos.com/multilingual_L-12_H-768_A-12.tar.gz",
    "bert-base-multilingual-cased": "https://bert-models.bj.bcebos.com/multi_cased_L-12_H-768_A-12.tar.gz",
    "bert-base-chinese": "https://bert-models.bj.bcebos.com/chinese_L-12_H-768_A-12.tar.gz",
}

def swish(x):
    return x * F.sigmoid(x)

ACT2FN = {
    "gelu": F.gelu,
    "relu": F.relu,
    "swish": swish
}

class BertConfig(object):
    """Configuration class to store the configuration of a `BertModel`."""
    def __init__(self,
                 vocab_size_or_config_json_file=30522,
                 hidden_size=768,
                 num_hidden_layers=12,
                 num_attention_heads=12,
                 intermediate_size=3072,
                 hidden_act="gelu",
                 hidden_dropout_prob=0.1,
                 attention_probs_dropout_prob=0.1,
                 max_position_embeddings=512,
                 type_vocab_size=2,
                 initializer_range=0.02,
                 layer_norm_eps=1e-12,
                 **kwargs):
        """Constructs BertConfig.

        Args:
            vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`.
            hidden_size: Size of the encoder layers and the pooler layer.
            num_hidden_layers: Number of hidden layers in the Transformer encoder.
            num_attention_heads: Number of attention heads for each attention layer in
                the Transformer encoder.
            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
                layer in the Transformer encoder.
            hidden_act: The non-linear activation function (function or string) in the
                encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
            hidden_dropout_prob: The dropout probabilitiy for all fully connected
                layers in the embeddings, encoder, and pooler.
            attention_probs_dropout_prob: The dropout ratio for the attention
                probabilities.
            max_position_embeddings: The maximum sequence length that this model might
                ever be used with. Typically set this to something large just in case
                (e.g., 512 or 1024 or 2048).
            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
                `BertModel`.
            initializer_range: The sttdev of the truncated_normal_initializer for
                initializing all weight matrices.
            layer_norm_eps: The epsilon used by LayerNorm.
        """
        if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                        and isinstance(vocab_size_or_config_json_file, unicode)):
            with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader:
                json_config = json.loads(reader.read())
            for key, value in json_config.items():
                self.__dict__[key] = value
        elif isinstance(vocab_size_or_config_json_file, int):
            self.vocab_size = vocab_size_or_config_json_file
            self.hidden_size = hidden_size
            self.num_hidden_layers = num_hidden_layers
            self.num_attention_heads = num_attention_heads
            self.hidden_act = hidden_act
            self.intermediate_size = intermediate_size
            self.hidden_dropout_prob = hidden_dropout_prob
            self.attention_probs_dropout_prob = attention_probs_dropout_prob
            self.max_position_embeddings = max_position_embeddings
            self.type_vocab_size = type_vocab_size
            self.initializer_range = initializer_range
            self.layer_norm_eps = layer_norm_eps
        else:
            raise ValueError("First argument must be either a vocabulary size (int)"
                             "or the path to a pretrained model config file (str)")

    @classmethod
    def from_dict(cls, json_object):
        """Constructs a `BertConfig` from a Python dictionary of parameters."""
        config = cls(vocab_size_or_config_json_file=-1)
        for key, value in json_object.items():
            config.__dict__[key] = value
        return config

    @classmethod
    def from_json_file(cls, json_file):
        """Constructs a `BertConfig` from a json file of parameters."""
        with open(json_file, "r", encoding='utf-8') as reader:
            text = reader.read()
        return cls.from_dict(json.loads(text))

    def __repr__(self):
        return str(self.to_json_string())

    def to_dict(self):
        """Serializes this instance to a Python dictionary."""
        output = copy.deepcopy(self.__dict__)
        return output

    def to_json_string(self):
        """Serializes this instance to a JSON string."""
        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"

    def to_json_file(self, json_file_path):
        """ Save this instance to a json file."""
        with open(json_file_path, "w", encoding='utf-8') as writer:
            writer.write(self.to_json_string())

    def save_pretrained(self, save_directory):
        """ Save a configuration object to a directory, so that it
            can be re-loaded using the `from_pretrained(save_directory)` class method.
        """
        assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved"

        # If we save using the predefined names, we can load using `from_pretrained`
        output_config_file = os.path.join(save_directory, "config.json")

        self.to_json_file(output_config_file)

class BertEmbeddings(nn.Layer):
    """Construct the embeddings from word, position and token_type embeddings."""
    def __init__(self, config):
        super(BertEmbeddings, self).__init__()
        self.word_embeddings = nn.Embedding(
            config.vocab_size, config.hidden_size, padding_idx=0
        )
        self.position_embeddings = nn.Embedding(
            config.max_position_embeddings, config.hidden_size
        )
        self.token_type_embeddings = nn.Embedding(
            config.type_vocab_size, config.hidden_size
        )
        self.LayerNorm = BertLayerNorm(
            config.hidden_size, epsilon=config.layer_norm_eps
        )
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, input_ids, token_type_ids=None, position_ids=None):
        seq_length = input_ids.shape[1]
        if position_ids is None:
            position_ids = paddle.arange(seq_length, dtype='int64')
            position_ids = position_ids.unsqueeze(0).expand_as(input_ids)
        if token_type_ids is None:
            token_type_ids = paddle.zeros_like(input_ids)

        words_embeddings = self.word_embeddings(input_ids)
        position_embeddings = self.position_embeddings(position_ids)
        token_type_embeddings = self.token_type_embeddings(token_type_ids)

        embeddings = words_embeddings + position_embeddings + token_type_embeddings
        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings

class BertSelfAttention(nn.Layer):
    def __init__(self, config):
        super(BertSelfAttention, self).__init__()
        if config.hidden_size % config.num_attention_heads != 0:
            raise ValueError(
                "The hidden size (%d) is not a multiple of the number of attention "
                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
            )
        self.output_attentions = config.output_attentions
        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(config.hidden_size, self.all_head_size)
        self.key = nn.Linear(config.hidden_size, self.all_head_size)
        self.value = nn.Linear(config.hidden_size, self.all_head_size)

        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)

    def transpose_for_scores(self, x):
        new_x_shape = x.shape[:-1] + [
            self.num_attention_heads,
            self.attention_head_size,
        ]
        x = x.reshape(shape=new_x_shape)
        return x.transpose((0, 2, 1, 3))

    def forward(self, hidden_states, attention_mask, head_mask=None):
        mixed_query_layer = self.query(hidden_states)
        mixed_key_layer = self.key(hidden_states)
        mixed_value_layer = self.value(hidden_states)

        query_layer = self.transpose_for_scores(mixed_query_layer)
        key_layer = self.transpose_for_scores(mixed_key_layer)
        value_layer = self.transpose_for_scores(mixed_value_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = paddle.matmul(query_layer, key_layer.transpose((0, 1, 3, 2)))
        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
        # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
        attention_scores = attention_scores + attention_mask

        # Normalize the attention scores to probabilities.
        attention_probs = nn.Softmax(axis=-1)(attention_scores)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        # Mask heads if we want to
        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = paddle.matmul(attention_probs, value_layer)
        context_layer = context_layer.transpose((0, 2, 1, 3))
        new_context_layer_shape = context_layer.shape[:-2] + [self.all_head_size,]
        context_layer = context_layer.reshape(shape=new_context_layer_shape)

        outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,)
        return outputs

class BertSelfOutput(nn.Layer):
    def __init__(self, config):
        super(BertSelfOutput, self).__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.LayerNorm = BertLayerNorm(config.hidden_size, epsilon=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

class BertAttention(nn.Layer):
    def __init__(self, config):
        super(BertAttention, self).__init__()
        self.self = BertSelfAttention(config)
        self.output = BertSelfOutput(config)

    def forward(self, input_tensor, attention_mask, head_mask=None):
        self_outputs = self.self(input_tensor, attention_mask, head_mask)
        attention_output = self.output(self_outputs[0], input_tensor)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

class BertIntermediate(nn.Layer):
    def __init__(self, config):
        super(BertIntermediate, self).__init__()
        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        if isinstance(config.hidden_act, str) or (
            sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)
        ):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

class BertOutput(nn.Layer):
    def __init__(self, config):
        super(BertOutput, self).__init__()
        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
        self.LayerNorm = BertLayerNorm(config.hidden_size, epsilon=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

class BertLayer(nn.Layer):
    def __init__(self, config):
        super(BertLayer, self).__init__()
        self.attention = BertAttention(config)
        self.intermediate = BertIntermediate(config)
        self.output = BertOutput(config)

    def forward(self, hidden_states, attention_mask, head_mask=None):
        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
        attention_output = attention_outputs[0]
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        outputs = (layer_output,) + attention_outputs[1:]  # add attentions if we output them
        return outputs

class BertEncoder(nn.Layer):
    def __init__(self, config):
        super(BertEncoder, self).__init__()
        self.output_attentions = config.output_attentions
        self.output_hidden_states = config.output_hidden_states
        self.layer = nn.LayerList([BertLayer(config) for _ in range(config.num_hidden_layers)])

    def forward(self, hidden_states, attention_mask, head_mask=None):
        all_hidden_states = ()
        all_attentions = ()
        for i, layer_module in enumerate(self.layer):
            if self.output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i])
            hidden_states = layer_outputs[0]

            if self.output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        # Add last layer
        if self.output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        outputs = (hidden_states,)
        if self.output_hidden_states:
            outputs = outputs + (all_hidden_states,)
        if self.output_attentions:
            outputs = outputs + (all_attentions,)
        return outputs  # outputs, (hidden states), (attentions)

class BertPooler(nn.Layer):
    def __init__(self, config):
        super(BertPooler, self).__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.activation = nn.Tanh()

    def forward(self, hidden_states):
        # We "pool" the model by simply taking the hidden state corresponding
        # to the first token.
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output

class BertPreTrainedModel(nn.Layer):
    """ An abstract class to handle weights initialization and a simple interface
    for dowloading and loading pretrained models."""
    def __init__(self, config, *inputs, **kwargs):
        super(BertPreTrainedModel, self).__init__()
        if not isinstance(config, BertConfig):
            raise ValueError(
                "Parameter config in `{}(config)` should be an instance of class `BertConfig`. "
                "To create a model from a Google pretrained model use "
                "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format(
                    self.__class__.__name__, self.__class__.__name__
                )
            )
        # Save config in model
        self.config = config

    def init_weights(self, layer):
        """ Initialize the weights."""
        if isinstance(layer, (nn.Linear, nn.Embedding)):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            layer.weight.set_value(
                paddle.tensor.normal(mean=0.0, std=self.config.initializer_range,
                                     shape=layer.weight.shape))

    @classmethod
    def from_pretrained(
            cls,
            pretrained_model_name_or_path,
            config,
            default_gpu=True,
            state_dict=None,
            cache_dir=None,
            from_tf=False,
            *inputs,
            **kwargs
    ):
        """
        Instantiate a BertPreTrainedModel from a pre-trained model file or a paddle state dict.
        Download and cache the pre-trained model file if needed.
        Params:
            pretrained_model_name_or_path: either:
                - a str with the name of a pre-trained model to load selected in the list of:
                    . `bert-base-uncased`
                    . `bert-large-uncased`
                    . `bert-base-cased`
                    . `bert-large-cased`
                    . `bert-base-multilingual-uncased`
                    . `bert-base-multilingual-cased`
                    . `bert-base-chinese`
                - a path or url to a pretrained model archive containing:
                    . `bert_config.json` a configuration file for the model
                    . `pytorch_model.bin` a PyTorch dump of a BertForPreTraining instance
                - a path or url to a pretrained model archive containing:
                    . `bert_config.json` a configuration file for the model
                    . `model.chkpt` a TensorFlow checkpoint
            from_tf: should we load the weights from a locally saved TensorFlow checkpoint
            cache_dir: an optional path to a folder in which the pre-trained models will be cached.
            state_dict: an optional state dictionnary (collections.OrderedDict object) to use instead of Google pre-trained models
            *inputs, **kwargs: additional input for the specific Bert class
                (ex: num_labels for BertForSequenceClassification)
        """
        CONFIG_NAME = "bert_config.json"
        WEIGHTS_NAME = "paddle_model.bin"
        TF_WEIGHTS_NAME = "model.ckpt"

        if pretrained_model_name_or_path in PRETRAINED_MODEL_ARCHIVE_MAP:
            archive_file = PRETRAINED_MODEL_ARCHIVE_MAP[pretrained_model_name_or_path]
        else:
            archive_file = pretrained_model_name_or_path

        # redirect to the cache, if necessary
        try:
            resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir)
        except EnvironmentError:
            logger.error(
                "Model name '{}' was not found in model name list ({}). "
                "We assumed '{}' was a path or url but couldn't find any file "
                "associated to this path or url.".format(
                    pretrained_model_name_or_path,
                    ", ".join(PRETRAINED_MODEL_ARCHIVE_MAP.keys()),
                    archive_file,
                )
            )
            return None

        if resolved_archive_file == archive_file:
            logger.info("loading archive file {}".format(archive_file))
        else:
            logger.info(
                "loading archive file {} from cache at {}".format(
                    archive_file, resolved_archive_file
                )
            )

        tempdir = None
        if os.path.isdir(resolved_archive_file) or from_tf:
            serialization_dir = resolved_archive_file
        elif resolved_archive_file[-3:] == 'bin':
            serialization_dir = '/'.join(resolved_archive_file.split('/')[:-1])
            WEIGHTS_NAME = resolved_archive_file.split('/')[-1]
        else:
            # Extract archive to temp dir
            tempdir = tempfile.mkdtemp()
            logger.info(
                "extracting archive file {} to temp dir {}".format(
                    resolved_archive_file, tempdir
                )
            )
            with tarfile.open(resolved_archive_file, "r:gz") as archive:
                archive.extractall(tempdir)
            serialization_dir = tempdir

        # config
        if default_gpu:
            logger.info("Model config {}".format(config))

        # Instantiate model.
        model = cls(config, *inputs, **kwargs)
        if state_dict is None and not from_tf:
            weights_path = os.path.join(serialization_dir, WEIGHTS_NAME)
            state_dict = paddle.load(
                weights_path,
            )
            if 'state_dict' in dir(state_dict):
                state_dict = state_dict.state_dict()

        if tempdir:
            # Clean up temp dir
            shutil.rmtree(tempdir)

        # Load from a PyTorch state_dict
        old_keys = []
        new_keys = []
        for key in state_dict.keys():
            new_key = None
            if "gamma" in key:
                new_key = key.replace("gamma", "weight")
            if "beta" in key:
                new_key = key.replace("beta", "bias")
            if new_key:
                old_keys.append(key)
                new_keys.append(new_key)
        for old_key, new_key in zip(old_keys, new_keys):
            state_dict[new_key] = state_dict.pop(old_key)

        missing_keys = []
        unexpected_keys = []
        error_msgs = []
        # copy state_dict so _load_from_state_dict can modify it
        metadata = getattr(state_dict, "_metadata", None)
        state_dict = state_dict.copy()
        if metadata is not None:
            state_dict._metadata = metadata

        def load(module, prefix=""):
            local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
            module._load_from_state_dict
            module._set_state_dict
            module.set_state_dict(
                state_dict,
                prefix,
                local_metadata,
                True,
                missing_keys,
                unexpected_keys,
                error_msgs,
            )
            for name, child in module._modules.items():
                if child is not None:
                    load(child, prefix + name + ".")

        start_prefix = ""
        if not hasattr(model, "bert") and any(
            s.startswith("bert.") for s in state_dict.keys()
        ):
            start_prefix = "bert."

        load(model, prefix=start_prefix)
        if len(missing_keys) > 0 and default_gpu:
            logger.info(
                "Weights of {} not initialized from pretrained model: {}".format(
                    model.__class__.__name__, missing_keys
                )
            )

        if len(unexpected_keys) > 0 and default_gpu:
            logger.info(
                "Weights from pretrained model not used in {}: {}".format(
                    model.__class__.__name__, unexpected_keys
                )
            )

        if len(error_msgs) > 0 and default_gpu:
            raise RuntimeError(
                "Error(s) in loading state_dict for {}:\n\t{}".format(
                    model.__class__.__name__, "\n\t".join(error_msgs)
                )
            )
        return model