import copy
import json
import os
import ssl
import time
import urllib.parse
from datetime import datetime
from threading import Lock, Thread

import websocket

from ..config import Config, Settings
from ..onap.audit import Audit
from ..utils import Utils
from .pdp_consts import MATCHING_CONDITIONS, POLICY_NAME, POLICY_VERSION

LOADED_POLICIES = 'loadedPolicies'
REMOVED_POLICIES = 'removedPolicies'
POLICY_VER = 'versionNo'
POLICY_MATCHES = 'matches'

_LOGGER = Utils.get_logger(__file__)

class PolicyListener(Thread):
    """web-socket to PolicyEngine"""
    PDP_API_FOLDER = os.path.basename(os.path.dirname(os.path.realpath(__file__)))
    WS_STARTED = "started"
    WS_START_COUNT = "start_count"
    WS_CLOSE_COUNT = "close_count"
    WS_ERROR_COUNT = "error_count"
    WS_PONG_COUNT = "pong_count"
    WS_MESSAGE_COUNT = "message_count"
    WS_MESSAGE_TIMESTAMP = "message_timestamp"
    WS_STATUS = "status"
    WS_PING_INTERVAL_DEFAULT = 30
    WEB_SOCKET_HEALTH = "web_socket_health"

    def __init__(self, audit, policy_updater):
        """web-socket inside the thread to receive policy notifications from PolicyEngine"""
        Thread.__init__(self, name="policy_receiver", daemon=True)

        self._policy_updater = policy_updater
        self._lock = Lock()
        self._keep_running = True
        self._settings = Settings(Config.FIELD_POLICY_ENGINE)

        self._sleep_before_restarting = 5
        self._web_socket_url = None
        self._web_socket_sslopt = None
        self._tls_wss_ca_mode = None
        self._web_socket = None
        self._ws_ping_interval_in_secs = PolicyListener.WS_PING_INTERVAL_DEFAULT
        self._web_socket_health = {
            PolicyListener.WS_START_COUNT: 0,
            PolicyListener.WS_CLOSE_COUNT: 0,
            PolicyListener.WS_ERROR_COUNT: 0,
            PolicyListener.WS_PONG_COUNT: 0,
            PolicyListener.WS_MESSAGE_COUNT: 0,
            PolicyListener.WS_STATUS: "created"
        }

        Audit.register_item_health(PolicyListener.WEB_SOCKET_HEALTH, self._get_health)
        self.reconfigure(audit)

    def reconfigure(self, audit):
        """configure and reconfigure the web-socket"""
        with self._lock:
            _LOGGER.info(audit.info("web_socket_health {}".format(
                json.dumps(self._get_health(), sort_keys=True))))
            self._sleep_before_restarting = 5
            self._settings.set_config(Config.discovered_config)
            changed, config = self._settings.get_by_key(Config.FIELD_POLICY_ENGINE)

            if not changed:
                self._settings.commit_change()
                return False

            prev_web_socket_url = self._web_socket_url
            prev_web_socket_sslopt = self._web_socket_sslopt
            prev_ws_ping_interval_in_secs = self._ws_ping_interval_in_secs

            self._web_socket_sslopt = None

            resturl = urllib.parse.urljoin(config.get("url", "").lower().rstrip("/") + "/",
                                           config.get("path_notifications", "/pdp/notifications"))

            self._tls_wss_ca_mode = config.get(Config.TLS_WSS_CA_MODE)

            self._ws_ping_interval_in_secs = config.get(Config.WS_PING_INTERVAL_IN_SECS)
            if not self._ws_ping_interval_in_secs or self._ws_ping_interval_in_secs < 60:
                self._ws_ping_interval_in_secs = PolicyListener.WS_PING_INTERVAL_DEFAULT

            if resturl.startswith("https:"):
                self._web_socket_url = resturl.replace("https:", "wss:")

                verify = Config.get_tls_verify(self._tls_wss_ca_mode)
                if verify is False:
                    self._web_socket_sslopt = {'cert_reqs': ssl.CERT_NONE}
                elif verify is True:
                    pass
                else:
                    self._web_socket_sslopt = {'ca_certs': verify}

            else:
                self._web_socket_url = resturl.replace("http:", "ws:")

            log_changed = (
                "changed web_socket_url(%s) or tls_wss_ca_mode(%s)"
                " or ws_ping_interval_in_secs(%s): %s" %
                (self._web_socket_url, self._tls_wss_ca_mode, self._ws_ping_interval_in_secs,
                 self._settings))
            if (self._web_socket_url == prev_web_socket_url
                    and Utils.are_the_same(prev_web_socket_sslopt, self._web_socket_sslopt)
                    and prev_ws_ping_interval_in_secs == self._ws_ping_interval_in_secs):
                _LOGGER.info(audit.info("not {}".format(log_changed)))
                self._settings.commit_change()
                return False

            _LOGGER.info(audit.info(log_changed))
            self._settings.commit_change()

        self._stop_notifications()
        return True

    def run(self):
        """listen on web-socket and pass the policy notifications to policy-updater"""
        _LOGGER.info("starting policy_receiver...")
        websocket.enableTrace(True)
        restarting = False
        while True:
            if not self._get_keep_running():
                break

            self._stop_notifications()

            if restarting:
                with self._lock:
                    sleep_before_restarting = self._sleep_before_restarting
                _LOGGER.info(
                    "going to sleep for %s secs before restarting policy-notifications",
                    sleep_before_restarting)

                time.sleep(sleep_before_restarting)
                if not self._get_keep_running():
                    break

            with self._lock:
                web_socket_url = self._web_socket_url
                sslopt = copy.deepcopy(self._web_socket_sslopt)
                tls_wss_ca_mode = self._tls_wss_ca_mode
                ws_ping_interval_in_secs = self._ws_ping_interval_in_secs

            _LOGGER.info(
                "connecting to policy-notifications at %s with sslopt(%s) tls_wss_ca_mode(%s)"
                " ws_ping_interval_in_secs(%s)",
                web_socket_url, json.dumps(sslopt), tls_wss_ca_mode, ws_ping_interval_in_secs)

            self._web_socket = websocket.WebSocketApp(
                web_socket_url,
                on_open=self._on_ws_open,
                on_message=self._on_pdp_message,
                on_close=self._on_ws_close,
                on_error=self._on_ws_error,
                on_pong=self._on_ws_pong
            )

            _LOGGER.info("waiting for policy-notifications...")
            self._web_socket.run_forever(sslopt=sslopt, ping_interval=ws_ping_interval_in_secs)
            restarting = True

        Audit.register_item_health(PolicyListener.WEB_SOCKET_HEALTH)
        _LOGGER.info("exit policy-receiver")

    def _get_keep_running(self):
        """thread-safe check whether to continue running"""
        with self._lock:
            keep_running = self._keep_running
        return keep_running

    def _stop_notifications(self):
        """close the web-socket == stops the notification service if running."""
        with self._lock:
            if self._web_socket and self._web_socket.sock and self._web_socket.sock.connected:
                self._web_socket.close()
                _LOGGER.info("stopped receiving notifications from PDP")

    def _on_pdp_message(self, *args):
        """received the notification from PDP"""
        self._web_socket_health[PolicyListener.WS_MESSAGE_COUNT] += 1
        self._web_socket_health[PolicyListener.WS_MESSAGE_TIMESTAMP] = str(datetime.utcnow())
        try:
            message = args and args[-1]
            _LOGGER.info("Received notification message: %s", message)
            _LOGGER.info("web_socket_health %s", json.dumps(self._get_health(), sort_keys=True))
            if not message:
                return
            message = json.loads(message)

            if not message or not isinstance(message, dict):
                _LOGGER.warning("unexpected message from PDP: %s", json.dumps(message))
                return

            policies_updated = [
                {POLICY_NAME: policy.get(POLICY_NAME),
                 POLICY_VERSION: policy.get(POLICY_VER),
                 MATCHING_CONDITIONS: policy.get(POLICY_MATCHES, {})}
                for policy in message.get(LOADED_POLICIES, [])
            ]

            policies_removed = [
                {POLICY_NAME: removed_policy.get(POLICY_NAME),
                 POLICY_VERSION: removed_policy.get(POLICY_VER)}
                for removed_policy in message.get(REMOVED_POLICIES, [])
            ]

            if not policies_updated and not policies_removed:
                _LOGGER.info("no policy updated or removed")
                return

            self._policy_updater.policy_update(policies_updated, policies_removed)
        except Exception as ex:
            error_msg = "crash {} {} at {}: {}".format(type(ex).__name__, str(ex),
                                                       "on_pdp_message", json.dumps(message))

            _LOGGER.exception(error_msg)

    def _on_ws_error(self, error):
        """report an error"""
        _LOGGER.exception("policy-notification error %s", str(error))
        self._sleep_before_restarting = 60 if isinstance(error, ssl.SSLError) else 5

        self._web_socket_health[PolicyListener.WS_STATUS] = "error"
        self._web_socket_health[PolicyListener.WS_ERROR_COUNT] += 1
        self._web_socket_health["last_error"] = {
            "error": str(error), "timestamp": str(datetime.utcnow())
        }
        _LOGGER.info("web_socket_health %s", json.dumps(self._get_health(), sort_keys=True))

    def _on_ws_close(self, code, reason):
        """restart web-socket on close"""
        self._web_socket_health["last_closed"] = str(datetime.utcnow())
        self._web_socket_health[PolicyListener.WS_STATUS] = "closed"
        self._web_socket_health[PolicyListener.WS_CLOSE_COUNT] += 1
        _LOGGER.info(
            "lost connection(%s, %s) to PDP web_socket_health %s",
            code, reason, json.dumps(self._get_health(), sort_keys=True))

    def _on_ws_open(self):
        """started web-socket"""
        self._web_socket_health[PolicyListener.WS_STATUS] = PolicyListener.WS_STARTED
        self._web_socket_health[PolicyListener.WS_START_COUNT] += 1
        self._web_socket_health[PolicyListener.WS_STARTED] = datetime.utcnow()
        _LOGGER.info("opened connection to PDP web_socket_health %s",
                     json.dumps(self._get_health(), sort_keys=True))

    def _on_ws_pong(self, pong):
        """pong = response to pinging the server of the web-socket"""
        self._web_socket_health[PolicyListener.WS_PONG_COUNT] += 1
        _LOGGER.info(
            "pong(%s) from connection to PDP web_socket_health %s",
            pong, json.dumps(self._get_health(), sort_keys=True))

    def _get_health(self):
        """returns the healthcheck of the web-socket as json"""
        web_socket_health = copy.deepcopy(self._web_socket_health)
        web_socket_health[Config.WS_PING_INTERVAL_IN_SECS] = self._ws_ping_interval_in_secs
        started = web_socket_health.get(PolicyListener.WS_STARTED)
        if started:
            web_socket_health[PolicyListener.WS_STARTED] = str(started)
            web_socket_health["uptime"] = str(datetime.utcnow() - started)
        return web_socket_health

    def shutdown(self, audit):
        """Shutdown the policy_listener"""
        _LOGGER.info(audit.info("shutdown policy_listener"))
        with self._lock:
            self._keep_running = False

        self._stop_notifications()

        if self.is_alive():
            self.join()


import random
import string
import tensorflow as tf
import numpy as np
import seq2seq as seq2seq
import data
from corpus_utils import LanguageIndex, tokenize_sentence

# def get_ELMo_embeddings():
#     url = "https://tfhub.dev/google/elmo/2"
#     elmo = hub.Module(url)
#     return elmo

def max_length(tensor):
    return max(len(t) for t in tensor)

def load_trained_model(batch_size, embedding_dim, units, optimizer, empty_token, unknown_token):
    questions, answers = data.load_conv_text()

    inp_lang = LanguageIndex(
        questions, empty_token=empty_token, unknown_token=unknown_token)
    targ_lang = LanguageIndex(
        answers, empty_token=empty_token, unknown_token=unknown_token)

    checkpoint_dir = './training_checkpoints'
    vocab_inp_size = len(inp_lang.word2idx)
    vocab_tar_size = len(targ_lang.word2idx)
    model = seq2seq.Seq2Seq(
        vocab_inp_size, vocab_tar_size, embedding_dim, units, batch_size, inp_lang, targ_lang,
        use_GloVe=True,
        mode=seq2seq.BASIC,
        use_bilstm=True)
    checkpoint = tf.train.Checkpoint(optimizer=optimizer, seq2seq=model)
    # restoring the latest checkpoint in checkpoint_dir
    checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))

    return model


#!/usr/bin/env python

#Importing Libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

#Importing Dataset and Segregating Independent & Dependent Variables
dataset = pd.read_csv("../Data/3_50Startups.csv")
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values

#Taking Care of Columns with Missing Data
#Not Applicable

#Encoding Categorical Data
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
labelencoder_State = LabelEncoder()
labelencoder_State = labelencoder_State.fit(X[:, 3])
X[:, 3] = labelencoder_State.transform(X[:, 3])
column_transformer = ColumnTransformer([("ohe", OneHotEncoder(), [3])], remainder = "passthrough")
X = np.array(column_transformer.fit_transform(X), dtype = np.int)

#Avoiding Dummy Variable Trap
X = X[:, 1:]                #Need not specifically code as MLR algorithm automatically takes care of it

#Splitting Dataset into Training & Testing Set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

#Fitting the Training Set
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train, y_train)

#Modelling Prediction Using Training Set
train_pred = regressor.predict(X_train)

#Validating Prediction Using Testing Set
test_pred = regressor.predict(X_test)

'''
#Comparing the Actual Values with the Predicted Values
print("Actual Result for X_test : \n")
print(y_test)
print("\n")
print("Predicted Values for X_test : \n")
print(test_pred)
'''

#Building Optimal Model Using Backward Elimination
import statsmodels.api as sm
##MLR ==> y = b0    + b1.X1 + b2.X2 + .... + bn.Xn
###    ==> y = b0.X0 + b1.X1 + b2.X2 + .... + bn.Xn where X0 = 1
##Insert X0 (a column of 1s in the beginning of X
##This manipulation is automatically taken care of by LinearRegression(). However, statsmodel.formula.api needs this to be done manually.
X = np.append(arr = np.ones((50, 1)).astype(int), values = X, axis = 1)  #Adding intercept i.e column of ones

##Manually finding the most significant Independent Variables with P < 0.05
'''
##Fitting the model with all possible predictors i.e., columns 0, 1, 2, 3 ...
##X : Original Matrix of Features
##X_optimal : Matrix of Optimal Features after eliminating insignifcant columns using Backward Elimination
X_optimal = X[:, [0, 1, 2, 3, 4, 5]]
regressor_OLS = sm.OLS(endog = y, exog = X_optimal).fit()
regressor_OLS.summary()     #Repeat the last three steps to remove the columns with P > 0.05 till identifying only the most significant I.Vs
'''

#Automatically finding the most significant Independent Variables with P < 0.05
def backwardElimination(sl, x_opt):
    numColumns = len(x_opt[0])          #Number of Columns
    for i in range(0, numColumns):
        regressor_OLS = sm.OLS(endog = y, exog = x_opt).fit()
        maxPvalue = max(regressor_OLS.pvalues).astype(float)

        if maxPvalue > sl:
            for j in range(0, numColumns - i):
                if(regressor_OLS.pvalues[j].astype(float) == maxPvalue):
                        x_opt = np.delete(x_opt, j, 1)
            print(regressor_OLS.summary())
    return x_opt

sig_level = 0.05
X_optimal = X[:, [0, 1, 2, 3, 4, 5]]
X_modelled = backwardElimination(sig_level, X_optimal)

#Build automatic Backward Elimination Model using P-values and Adjusted R squares values
print("\n\n")
print("***Build Automatic Backward Elimination Model Using P-values and Adjusted-R-Squares values***")


"""add Comment.reply

Revision ID: 9b356a815da2
Revises: df35517b3f21
Create Date: 2019-11-26 15:33:59.360507

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '9b356a815da2'
down_revision = 'df35517b3f21'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('comments', sa.Column('reply_id', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'comments', 'comments', ['reply_id'], ['id'])
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'comments', type_='foreignkey')
    op.drop_column('comments', 'reply_id')
    # ### end Alembic commands ###


import os
from re import S
import cv2
import glob
import numpy as np
from moviepy.editor import *
import time
import pickle

def video_olustur(folders):
    #Amacım .mp4 formatından biten videoların tespiti ve bunların  jpg formatına çevirerek
    # (1920,1080) ve 5 fps formatlarında output videosu oluşturmak.
  videonames_list = []
  for folder in folders:
    for f in glob.glob(folder+'/*.mp4'):
      videonames_list.append(f)
      print(f)

  print('There are {} videos in Folder'.format(len(videonames_list)))

  count = 0
  for i in range(0,len(videonames_list)):
   video_data = videonames_list[i]
   video = cv2.VideoCapture(video_data)
   success = True
   while success:
    success,image = video.read()
    name = 'C:/Users/DELL/Desktop/Ornek/Resim/'+str(count)+'.jpg'
    if success == True:
      cv2.imwrite(name,image)
      print('Frame {} Extracted Successfully'.format(count))
      count+=1
    else:
      i = i+1
    i = i+1
  print('\n\n\nVideo {} Extracted Successfully\n\n\n'.format(video_data))

  image_folder = 'Resim'
  video_name = 'video.avi'

  images = [img for img in os.listdir(image_folder) if img.endswith(".jpg")]
  frame = cv2.imread(os.path.join(image_folder, images[0]))
#   height, width, layers = frame.shape

#Video oluşturuyoruz
  fourcc = cv2.VideoWriter_fourcc(*'XVID')
  video=cv2.VideoWriter('output6.avi',fourcc, 5, (1920,1080))

  for image in images:
    video.write(cv2.imread(os.path.join(image_folder, image)))
  return video

if __name__ == "__main__":

    yol=glob.glob('C:/Users/DELL/Desktop/Ornek')
    folder=video_olustur(yol)
    time.sleep(10)
    cap = cv2.VideoCapture('C:/Users/DELL/Desktop/Ornek/output6.avi')  # 0 olarak değiştirildi
    frame=[]
    color2=[]
    my_dict=dict()

    #number of frames 670
    for frame_indx in range(670) :
      while (cap.isOpened()):
       ret, frame = cap.read()
       if ret==True:
        hsv_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
        height, width, _ = frame.shape

        cx = int(width / 2)
        cy = int(height / 2)

     # Pick pixel value
        pixel_center = hsv_frame[cy, cx]
        hue_value = pixel_center[0]

        i=0
        color = "Undefined"
        if hue_value < 5:
          color = "KIRMIZI"
          color2.append(color)

        elif hue_value < 78:
          color = "YEŞİL"
          #print('Yeşil')
          color2.append(color)

        elif hue_value < 131:
           color = "MAVi"
           color2.append(color)

        else:
           color = "KIRMIZI"
          # print('Kırnızı')

    # print(color2)

        # pixel_center_bgr = frame[cy, cx]
        # b, g, r = int(pixel_center_bgr[0]), int(pixel_center_bgr[1]), int(pixel_center_bgr[2])

        # cv2.rectangle(frame, (cx - 220, 10), (cx + 200, 120), (255, 255, 255), -1)
        # cv2.putText(frame, color, (cx - 200, 100), 0, 3, (b, g, r), 5)
        # cv2.circle(frame, (cx, cy), 5, (25, 25, 25), 3)

        cv2.imshow("Frame", frame)
        key = cv2.waitKey(1)
        if key == 27:
          break
        else:

          break

#Bu kısımda color2 listesini dictionary çeviriyorum
    for index,value in enumerate(color2):
       my_dict[index] = value
    print(my_dict)

    with open('renk.pickle', 'wb') as handle:
       pickle.dump(my_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

    with open('renk.pickle', 'rb') as handle:
       b = pickle.load(handle)

    print(my_dict == b)
    cap.release()
# video.release()
    cv2.destroyAllWindows()


from django.shortcuts import render,get_object_or_404, get_list_or_404, redirect
from django.http import JsonResponse, HttpResponse
from django.utils.translation import ugettext as _
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from contract.models import Contract
from .models import Annotation
from .form import AnnotationForm
from account.models import User
from django.db import IntegrityError
from django.db.models import Q, Sum, Count, Prefetch
import os
import json

####### SERVICETYPE  ################

def annotation_save_form(request,form,template_name, data, user_created=None):
    if request.method == 'POST':
        if form.is_valid():
            obj = form.save(commit=False)
            if user_created:# Se cair aqui é EDIT
                obj.user_created = user_created
            else:# Se cair aqui é CREATE
                obj.user_created = request.user
            obj.user_updated = request.user
            obj.save()

            return redirect('annotation:url_annotations_list',data['contract'].slug)
        else:
            print("algo não está valido.")

    data['form'] = form
    return render(request,template_name,data)

@login_required
def annotation_create(request,contract):
    contract = get_object_or_404(Contract, slug=contract)
    template_name = 'annotation/form.html'
    data = {
            "title": _("Create Annotation"),
            "back":_("Back"),
            "save":_("Save"),
            "clear":_("Clear"),
            "contract": contract,
        }
    if request.method == 'POST':
        form = AnnotationForm(request.POST, request.FILES,contract=contract.id)
    else:
        form = AnnotationForm(contract=contract.id)

    return annotation_save_form(request, form, template_name, data)

@login_required
def annotation_edit(request, slug):
    annotation = get_object_or_404(Annotation, slug=slug)
    contract = get_object_or_404(Contract, slug=annotation.contract.slug)
    template_name='annotation/form.html'
    data = {
            "title": _("Edit"),
            "back":_("Back"),
            "save":_("Save"),
            "clear":_("Clear"),
            "contract": contract,
        }
    user_created = annotation.user_created # Esta linha faz com que o user_created não seja modificado, para mostrar quem criou esta pessoa
    if request.method == 'POST':
        form = AnnotationForm(request.POST, request.FILES, instance=annotation)
    else:
        form = AnnotationForm(instance=annotation)
    return annotation_save_form(request, form, template_name, data, user_created=user_created)

@login_required
def annotations_list(request,contract):
    contract = get_object_or_404(Contract, slug=contract)
    template_name = "annotation/list.html"
    annotations = Annotation.objects.filter(contract=contract.id)
    context = {
        'annotations': annotations,
        'contract': contract,
        'title': _("Registered Annotations"),
        'add': _("Add"),
        'vars': [_("Contract"), _("Provider")],
        'back': _("Back"),
    }
    return render(request,template_name,context)

@login_required
def annotation_detail(request, slug):
    template_name = "annotation/detail.html"
    annotation = get_object_or_404(Annotation,slug=slug)
    contract = get_object_or_404(Contract,slug=annotation.contract.slug)
    context = {
        'annotation': annotation,
        'contract': contract,
        'title': _("Detail Info"),
        'edit': _("Edit"),
        'list_all': _("List All"),
        'vars': [_("Contract"), _("Provider")],
    }
    return render(request, template_name, context)

@login_required
def annotation_delete(request, slug):
    annotation = get_object_or_404(Annotation, slug=slug)
    slug = annotation.contract.slug
    if request.method == 'POST':
       try:
           annotation.delete()
           messages.success(request, _('Completed successful.'))
           return redirect('annotation:url_annotations_list', contract=slug)
       except IntegrityError:
           messages.warning(request, _('You cannot delete. This annotation has an existing deal.'))
           return redirect('annotation:url_annotations_list', contract=slug)

@login_required
def annotation_delete_all(request):
    marc = 0
    if request.method == "POST":
        context = request.POST["checkbox_selected"].split(",")
        context = [str(x) for x in context]
        if context:
            contract_slug = Annotation.objects.prefetch_related('contract').values('contract__slug').distinct()
            contract_slug = contract_slug[0]['contract__slug']
            b = Annotation.objects.filter(slug__in=context)
            for i in b:
                try:
                    i.delete()
                except IntegrityError:
                    marc = 1
    if marc == 0:
        messages.success(request, _('Completed successful.'))
    else:
        messages.warning(request, _('You cannot delete. This annotation has an existing deal.'))

    return redirect('annotation:url_annotations_list', contract=contract_slug)

########### FIM SERVICETYPE ############################

# VIEW PARA TRADUZIR O DATATABLES. USO GERAL
def translate_datables_js(request):
    if request.method == "GET":
        module_dir = os.path.dirname(__file__)  # get current directory
        file_path = os.path.join(module_dir, 'templates/default')
        translate = ""
        with open(file_path+"/translate_data_tables-"+request.LANGUAGE_CODE+".json", 'r') as arquivo:
            for linha in arquivo:
                translate += linha
        obj = json.loads(translate)
    return JsonResponse(obj)


import json
from copy import deepcopy
from datetime import datetime, timedelta

import mock
from flask import url_for
from invenio_accounts.testutils import login_user_via_session
from utils import VerifyRecordPermissionPatch, flush_index, get_json, \
    mock_response, postdata

from rero_ils.modules.commons.identifiers import IdentifierType
from rero_ils.modules.documents.api import DocumentsSearch
from rero_ils.modules.documents.utils import clean_text, get_remote_cover
from rero_ils.modules.documents.views import can_request, \
    record_library_pickup_locations
from rero_ils.modules.operation_logs.api import OperationLogsSearch
from rero_ils.modules.utils import get_ref_for_pid

def test_documents_newacq_filters(app, client,
                                  system_librarian_martigny,
                                  rero_json_header, document,
                                  holding_lib_martigny, holding_lib_saxon,
                                  loc_public_saxon,
                                  item_lib_martigny_data,
                                  ):
    login_user_via_session(client, system_librarian_martigny.user)

    # compute useful date
    today = datetime.today()
    past = (today - timedelta(days=1)).strftime('%Y-%m-%d')
    future = (today + timedelta(days=10)).strftime('%Y-%m-%d')
    future_1 = (today + timedelta(days=11)).strftime('%Y-%m-%d')
    today = today.strftime('%Y-%m-%d')

    # Add a new items with acq_date
    new_acq1 = deepcopy(item_lib_martigny_data)
    new_acq1['pid'] = 'itemacq1'
    new_acq1['acquisition_date'] = today
    res, data = postdata(client, 'invenio_records_rest.item_list', new_acq1)
    assert res.status_code == 201

    new_acq2 = deepcopy(item_lib_martigny_data)
    new_acq2['pid'] = 'itemacq2'
    new_acq2['acquisition_date'] = future
    new_acq2['location']['$ref'] = get_ref_for_pid('loc', loc_public_saxon.pid)
    res, data = postdata(client, 'invenio_records_rest.item_list', new_acq2)
    assert res.status_code == 201

    # check item creation and indexation
    doc_list = url_for(
        'invenio_records_rest.doc_list',
        view='global', pid='doc1'
    )
    res = client.get(doc_list, headers=rero_json_header)
    data = get_json(res)
    assert len(data['hits']['hits']) == 1
    data = data['hits']['hits'][0]['metadata']
    assert len(data['holdings']) == 2
    assert len(data['holdings'][0]['items']) == 1
    assert len(data['holdings'][1]['items']) == 1

    # check new_acquisition filters
    #   --> For org2, there is no new acquisition
    doc_list = url_for(
        'invenio_records_rest.doc_list',
        view='global',
        new_acquisition=':',
        organisation='org2'
    )
    res = client.get(doc_list, headers=rero_json_header)
    data = get_json(res)
    assert data['hits']['total']['value'] == 0

    #   --> for org1, there is 1 document with 2 new acquisition items
    doc_list = url_for(
        'invenio_records_rest.doc_list',
        view='global',
        new_acquisition='{0}:{1}'.format(past, future_1),
        organisation='org1'
    )
    res = client.get(doc_list, headers=rero_json_header)
    data = get_json(res)
    assert data['hits']['total']['value'] == 1
    assert len(data['hits']['hits'][0]['metadata']['holdings']) == 2

    #   --> for lib2, there is 1 document with 1 new acquisition items
    doc_list = url_for(
        'invenio_records_rest.doc_list',
        view='global',
        new_acquisition='{0}:{1}'.format(past, future_1),
        library='lib2'
    )
    res = client.get(doc_list, headers=rero_json_header)
    data = get_json(res)
    assert data['hits']['total']['value'] == 1

    #   --> for loc3, there is 1 document with 1 new acquisition items
    doc_list = url_for(
        'invenio_records_rest.doc_list',
        view='global',
        new_acquisition='{0}:{1}'.format(past, future_1),
        location='loc3'
    )
    res = client.get(doc_list, headers=rero_json_header)
    data = get_json(res)
    assert data['hits']['total']['value'] == 1

    #   --> for loc3, there is no document corresponding to range date
    doc_list = url_for(
        'invenio_records_rest.doc_list',
        view='global',
        new_acquisition='{0}:{1}'.format(past, today),
        location='loc3'
    )
    res = client.get(doc_list, headers=rero_json_header)
    data = get_json(res)
    assert data['hits']['total']['value'] == 0

@mock.patch('invenio_records_rest.views.verify_record_permission',
            mock.MagicMock(return_value=VerifyRecordPermissionPatch))
def test_documents_facets(
    client, document, document2_ref, ebook_1, ebook_2, ebook_3, ebook_4,
    item_lib_martigny, rero_json_header
):
    """Test record retrieval."""
    # STEP#1 :: CHECK FACETS ARE PRESENT INTO SEARCH RESULT
    url = url_for('invenio_records_rest.doc_list', view='global')
    res = client.get(url, headers=rero_json_header)
    data = get_json(res)
    facet_keys = [
        'document_type', 'author', 'language', 'subject_no_fiction',
        'subject_fiction', 'genreForm', 'intendedAudience',
        'year', 'status'
    ]
    assert all(key in data['aggregations'] for key in facet_keys)

    params = {'view': 'global', 'facets': ''}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url, headers=rero_json_header)
    data = get_json(res)
    assert not data['aggregations']

    params = {'view': 'global', 'facets': 'document_type'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url, headers=rero_json_header)
    data = get_json(res)
    assert list(data['aggregations'].keys()) == ['document_type']

    params = {'view': 'org1', 'facets': 'document_type'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url, headers=rero_json_header)
    data = get_json(res)
    assert list(data['aggregations'].keys()) == ['document_type']

    # test the patch that the library facet is computed by the serializer
    params = {'view': 'org1', 'facets': 'document_type,library,author'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url, headers=rero_json_header)
    data = get_json(res)
    aggs = data['aggregations']
    assert set(aggs.keys()) == {'document_type', 'library', 'author'}

    # TEST FILTERS
    # Each filter checks is a tuple. First tuple element is argument used to
    # call the API, second tuple argument is the number of document that
    # should be return by the API call.
    checks = [
        ({'view': 'global', 'author': 'Peter James'}, 2),
        ({'view': 'global', 'author': 'Great Edition'}, 1),
        ({'view': 'global', 'author': 'J.K. Rowling'}, 1),
        ({'view': 'global', 'author': ['Great Edition', 'Peter James']}, 1),
        ({'view': 'global', 'author': ['J.K. Rowling', 'Peter James']}, 0),
        # i18n facets
        ({'view': 'global', 'author': 'Nebehay, Christian Michael'}, 1),
        ({'view': 'global',
          'author': 'Nebehay, Christian Michael, 1909-2003', 'lang': 'de'}, 0),
        ({'view': 'global',
          'author': 'Nebehay, Christian Michael', 'lang': 'thl'}, 1),
    ]
    for params, value in checks:
        url = url_for('invenio_records_rest.doc_list', **params)
        res = client.get(url)
        data = get_json(res)
        assert data['hits']['total']['value'] == value

@mock.patch('invenio_records_rest.views.verify_record_permission',
            mock.MagicMock(return_value=VerifyRecordPermissionPatch))
def test_documents_organisation_facets(
    client, document, item_lib_martigny, rero_json_header
):
    """Test record retrieval."""
    list_url = url_for('invenio_records_rest.doc_list', view='global')

    res = client.get(list_url, headers=rero_json_header)
    data = get_json(res)
    aggs = data['aggregations']

    assert 'organisation' in aggs

@mock.patch('invenio_records_rest.views.verify_record_permission',
            mock.MagicMock(return_value=VerifyRecordPermissionPatch))
def test_documents_library_location_facets(
    client, document, org_martigny, item_lib_martigny, rero_json_header
):
    """Test record retrieval."""
    list_url = url_for('invenio_records_rest.doc_list', view='org1')

    res = client.get(list_url, headers=rero_json_header)
    data = get_json(res)
    aggs = data['aggregations']

    assert 'library' in aggs

    # Test if location sub-buckets exists under each Library hit
    for hit in aggs['library']['buckets']:
        assert 'location' in hit

@mock.patch('invenio_records_rest.views.verify_record_permission',
            mock.MagicMock(return_value=VerifyRecordPermissionPatch))
def test_documents_post_put_delete(
    client, document_chinese_data, json_header, rero_json_header
):
    """Test record retrieval."""
    # Create record / POST
    item_url = url_for('invenio_records_rest.doc_item', pid_value='4')
    list_url = url_for('invenio_records_rest.doc_list', q='pid:4')

    document_chinese_data['pid'] = '4'
    res, data = postdata(
        client,
        'invenio_records_rest.doc_list',
        document_chinese_data
    )

    assert res.status_code == 201

    # Check that the returned record matches the given data
    test_data = data['metadata']
    test_data.pop('sort_title', None)
    assert clean_text(test_data) == document_chinese_data

    res = client.get(item_url)
    assert res.status_code == 200
    data = get_json(res)

    test_data = data['metadata']
    test_data.pop('sort_title', None)
    assert clean_text(test_data) == document_chinese_data
    expected_title = [
        {
            '_text': '\u56fd\u9645\u6cd5 : subtitle (Chinese). '
                     'Part Number (Chinese), Part Name (Chinese) = '
                     'International law (Chinese) : '
                     'Parallel Subtitle (Chinese). '
                     'Parallel Part Number (Chinese), '
                     'Parallel Part Name (Chinese) = '
                     'Parallel Title 2 (Chinese) : '
                     'Parallel Subtitle 2 (Chinese)',
            'mainTitle': [
                    {'value': 'Guo ji fa'},
                    {
                        'value': '\u56fd\u9645\u6cd5',
                        'language': 'chi-hani'
                    }
            ],
            'subtitle': [
                {'value': 'subtitle (Latin)'},
                {
                    'value': 'subtitle (Chinese)',
                    'language': 'chi-hani'
                }
            ],
            'part': [{
                'partNumber': [
                    {'value': 'Part Number (Latin)'},
                    {
                        'value': 'Part Number (Chinese)',
                        'language': 'chi-hani'
                    }
                ],
                'partName': [
                    {'value': 'Part Name (Latin)'},
                    {
                        'language': 'chi-hani',
                        'value': 'Part Name (Chinese)'
                    }
                ]
            }],
            'type': 'bf:Title'
        },
        {
            'mainTitle': [
                {'value': 'International law (Latin)'},
                {
                    'value': 'International law (Chinese)',
                    'language': 'chi-hani'
                }
            ],
            'subtitle': [
                {'value': 'Parallel Subtitle (Latin)'},
                {
                    'value': 'Parallel Subtitle (Chinese)',
                    'language': 'chi-hani'
                }
            ],
            'part': [{
                'partNumber': [
                    {'value': 'Parallel Part Number (Latin)'},
                    {
                        'value': 'Parallel Part Number (Chinese)',
                        'language': 'chi-hani'
                    }
                ],
                'partName': [
                    {'value': 'Parallel Part Name (Latin)'},
                    {
                        'language': 'chi-hani',
                        'value': 'Parallel Part Name (Chinese)'
                    }
                ]
            }],

            'type': 'bf:ParallelTitle'
        },
        {
            'mainTitle': [
                {'value': 'Parallel Title 2 (Latin)'},
                {
                    'value': 'Parallel Title 2 (Chinese)',
                    'language': 'chi-hani'
                }
            ],
            'subtitle': [
                {'value': 'Parallel Subtitle 2 (Latin)'},
                {
                    'value': 'Parallel Subtitle 2 (Chinese)',
                    'language': 'chi-hani'
                }
            ],
            'type': 'bf:ParallelTitle'
        },
        {
            'mainTitle': [{'value': 'Guojifa'}],
            'type': 'bf:VariantTitle'
        }
    ]

    # Update record/PUT
    data = document_chinese_data
    res = client.put(
        item_url,
        data=json.dumps(data),
        headers=rero_json_header
    )
    assert res.status_code == 200
    # assert res.headers['ETag'] != '"{}"'.format(librarie.revision_id)

    # Check that the returned record matches the given data
    data = get_json(res)
    assert data['metadata']['title'] == expected_title
    assert data['metadata']['ui_title_variants'] == ['Guojifa']
    assert data['metadata']['ui_title_altgr'] == \
        ['Guo ji fa : subtitle (Latin). Part Number (Latin), Part Name (Latin)'
         ' = International law (Latin) : Parallel Subtitle (Latin).'
         ' Parallel Part Number (Latin), Parallel Part Name (Latin)'
         ' = Parallel Title 2 (Latin) : Parallel Subtitle 2 (Latin)']
    assert data['metadata']['ui_responsibilities'] == [
        '梁西原著主编, 王献枢副主编',
        'Liang Xi yuan zhu zhu bian, Wang Xianshu fu zhu bian'
    ]

    res = client.get(item_url)
    assert res.status_code == 200

    # Delete record/DELETE
    res = client.delete(item_url)
    assert res.status_code == 204

    res = client.get(item_url)
    assert res.status_code == 410

def test_documents_get_resolve_rero_json(
    client, document_ref, entity_person_data, rero_json_header,
):
    """Test record get with resolve and mimetype rero+json."""
    api_url = url_for('invenio_records_rest.doc_item', pid_value='doc2',
                      resolve='1')
    res = client.get(api_url, headers=rero_json_header)
    assert res.status_code == 200
    metadata = get_json(res).get('metadata', {})
    pid = metadata['contribution'][0]['entity']['pid']
    assert pid == entity_person_data['pid']

def test_document_can_request_view(
        client, item_lib_fully,
        loan_pending_martigny, document,
        patron_martigny,
        patron2_martigny,
        item_type_standard_martigny,
        circulation_policies,
        librarian_martigny,
        item_lib_martigny,
        item_lib_saxon,
        item_lib_sion,
        loc_public_martigny
):
    """Test can request on document view."""
    login_user_via_session(client, patron_martigny.user)

    with mock.patch(
        'rero_ils.modules.documents.views.current_user',
        patron_martigny.user
    ), mock.patch(
        'rero_ils.modules.documents.views.current_patrons',
        [patron_martigny]
    ):
        can, _ = can_request(item_lib_fully)
        assert can
        can, _ = can_request(item_lib_sion)
        assert not can

    with mock.patch(
        'rero_ils.modules.documents.views.current_user',
        patron2_martigny.user
    ), mock.patch(
        'rero_ils.modules.documents.views.current_patrons',
        [patron2_martigny]
    ):
        can, _ = can_request(item_lib_fully)
        assert not can

    picks = record_library_pickup_locations(item_lib_fully)
    assert len(picks) == 3

    picks = record_library_pickup_locations(item_lib_martigny)
    assert len(picks) == 3

def test_document_boosting(client, ebook_1, ebook_4):
    """Test document boosting."""
    list_url = url_for(
        'invenio_records_rest.doc_list',
        q='maison'
    )
    res = client.get(list_url)
    hits = get_json(res)['hits']
    assert hits['total']['value'] == 2
    data = hits['hits'][0]['metadata']
    assert data['pid'] == ebook_1.pid

    list_url = url_for(
        'invenio_records_rest.doc_list',
        q='autocomplete_title:maison AND' +
          'contribution.agent.authorized_access_point:James'
    )
    res = client.get(list_url)
    hits = get_json(res)['hits']
    assert hits['total']['value'] == 1
    data = hits['hits'][0]['metadata']
    assert data['pid'] == ebook_1.pid

@mock.patch('requests.Session.get')
def test_documents_resolve(
    mock_contributions_mef_get, client, mef_agents_url, loc_public_martigny,
    document_ref, entity_person_response_data
):
    """Test document detailed view with items filter."""
    res = client.get(url_for(
        'invenio_records_rest.doc_item',
        pid_value='doc2'
    ))
    assert res.json['metadata']['contribution'] == [{
        'entity': {
            '$ref': f'{mef_agents_url}/rero/A017671081',
            'pid': 'ent_pers',
        },
        'role': ['aut']
    }]
    assert res.status_code == 200

    mock_contributions_mef_get.return_value = mock_response(
        json_data=entity_person_response_data
    )
    res = client.get(url_for(
        'invenio_records_rest.doc_item',
        pid_value='doc2',
        resolve='1'
    ))
    assert res.json['metadata'][
        'contribution'][0]['entity']['authorized_access_point_fr']
    assert res.status_code == 200

def test_document_exclude_draft_records(client, document):
    """Test document exclude draft record."""
    list_url = url_for(
        'invenio_records_rest.doc_list',
        q='Lingliang'
    )
    res = client.get(list_url)
    hits = get_json(res)['hits']
    assert hits['total']['value'] == 1
    data = hits['hits'][0]['metadata']
    assert data['pid'] == document.get('pid')

    document['_draft'] = True
    document.update(document, dbcommit=True, reindex=True)

    list_url = url_for(
        'invenio_records_rest.doc_list',
        q='Lingliang'
    )
    res = client.get(list_url)
    hits = get_json(res)['hits']
    assert hits['total']['value'] == 0

    document['_draft'] = False
    document.update(document, dbcommit=True, reindex=True)

    list_url = url_for(
        'invenio_records_rest.doc_list',
        q='Lingliang'
    )
    res = client.get(list_url)
    hits = get_json(res)['hits']
    assert hits['total']['value'] == 1

@mock.patch('requests.get')
def test_get_remote_cover(mock_get_cover, app):
    """Test get remote cover."""
    mock_get_cover.return_value = mock_response(status=400)
    assert get_remote_cover('YYYYYYYYY') is None

    mock_get_cover.return_value = mock_response(
        content='thumb({'
                '    "success": true,'
                '    "image": "https://i.test.com/images/P/XXXXXXXXXX_.jpg"'
                '})'
    )
    cover = get_remote_cover('XXXXXXXXXX')
    assert cover == {
        'success': True,
        'image': 'https://i.test.com/images/P/XXXXXXXXXX_.jpg'
    }

def test_document_identifiers_search(client, document):
    """Test search on `identifiedBy` document."""

    def success(response_data):
        data = response_data['hits']
        return data['total']['value'] == 1 \
            and data['hits'][0]['metadata']['pid'] == document.pid

    def failure(response_data):
        return response_data['hits']['total']['value'] == 0

    # STEP#1 :: SEARCH FOR AN EXISTING IDENTIFIER
    #   Search for an existing encoded document identifier. The ISBN-13 is
    #   encoded into document data. Search on this specific value will return
    #   a record.
    params = {'identifiers': '(bf:Isbn)9782844267788'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    # STEP#2 :: SEARCH FOR AN ALTERNATIVE IDENTIFIER
    #   Search for the alternative of the encoded ISBN-13 value. During the
    #   document indexing process the corresponding ISBN-10 is appended to
    #   identifier list. A search on this value should return the same
    #   document. Additionally, search with hyphens to validate the specific
    #   identifier analyzer used for this field.
    params = {'identifiers': '(bf:Isbn)2-84426-778-5'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    # STEP#3 :: SEARCH WITH ONLY IDENTIFIER VALUE
    #   Search only about an identifier value without specified any identifier
    #   type.
    params = {'identifiers': 'R008745599'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    # STEP#4 :: SEARCH ON UNKNOWN IDENTIFIERS
    for id_value in ['dummy_identifiers', '(bf:Issn)9782844267788']:
        params = {'identifiers': id_value}
        url = url_for('invenio_records_rest.doc_list', **params)
        res = client.get(url)
        assert failure(get_json(res))

    # STEP#5 :: GROUPED SEARCH
    #   Use this filter in combination with other filter. In this test, the
    #   document isn't an harvested document, but it contains the correct
    #   specified identifier.
    params = {
        'identifiers': '(bf:Ean)9782844267788',
        'q': 'harvested:true'
    }
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert failure(get_json(res))

    params['q'] = 'harvested:false'
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    # STEP#6 :: SEARCH USING SHORTCUT ES KEYS
    #   'isbn' and 'issn' keys are added to the ES stored document. These key
    #   only contains the corresponding identifiers value ; but analyzer
    #   allows search using hyphens or not.
    url = url_for('invenio_records_rest.doc_list', q='isbn:2-84426-778-5')
    res = client.get(url)
    assert success(get_json(res))

    # STEP#7 :: WILDCARD SEARCH
    #    `identifiers` filter allow to search on a partial identifier string
    #    (only for the identifier value part).
    params = {'identifiers': 'R0087455*'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    params = {'identifiers': '(bf:Local)*87455*'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    params = {'identifiers': '*dummy_search*'}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert failure(get_json(res))

    # STEP#8 :: SEARCH WITH MULTIPLE IDENTIFIERS
    #    If we send multiple identifiers, an OR query will be used to search on
    #    each of them.
    params = {'identifiers': ['dummy', 'other_id', '(bf:Ean)9782844267788']}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    # STEP#9 :: SEARCH ABOUT INVALID EAN IDENTIFIER
    #    Ensure than if an invalid EAN identifier exists into the document
    #    metadata, this identifier is searchable anyway.
    original_data = deepcopy(document)
    document['identifiedBy'].append({
        'type': IdentifierType.EAN,
        'value': 'invalid_ean_identifier'
    })
    document.update(document, dbcommit=True, reindex=True)
    flush_index(DocumentsSearch.Meta.index)

    params = {'identifiers': ['(bf:Ean)invalid_ean_identifier']}
    url = url_for('invenio_records_rest.doc_list', **params)
    res = client.get(url)
    assert success(get_json(res))

    # RESET THE DOCUMENT
    document.update(original_data, dbcommit=True, reindex=True)

def test_document_current_library_on_request_parameter(
    app, db, client, system_librarian_martigny, lib_martigny,
    lib_martigny_bourg, document, json_header
):
    """Test for library assignment if the current_library parameter
    is present in the request."""
    login_user_via_session(client, system_librarian_martigny.user)

    # Assign library pid with current_librarian information
    document['copyrightDate'] = ['© 2023']
    doc_url = url_for('invenio_records_rest.doc_item', pid_value=document.pid)
    res = client.put(doc_url, data=json.dumps(document), headers=json_header)
    assert res.status_code == 200
    flush_index(OperationLogsSearch.Meta.index)
    oplg = next(OperationLogsSearch()
                .filter('term', record__type='doc')
                .filter('term', record__value=document.pid)
                .params(preserve_order=True)
                .sort({'date': 'desc'})
                .scan())
    assert oplg.library.value == lib_martigny.pid
    db.session.rollback()

    # Assign library pid with current_library request parameter
    document['copyrightDate'] = ['© 1971']
    doc_url = url_for(
        'invenio_records_rest.doc_item',
        pid_value=document.pid,
        current_library=lib_martigny_bourg.pid)
    res = client.put(doc_url, data=json.dumps(document), headers=json_header)
    assert res.status_code == 200
    flush_index(OperationLogsSearch.Meta.index)
    oplg = next(OperationLogsSearch()
                .filter('term', record__type='doc')
                .filter('term', record__value=document.pid)
                .params(preserve_order=True)
                .sort({'date': 'desc'})
                .scan())
    assert oplg.library.value == lib_martigny_bourg.pid
    db.session.rollback()


import builtins as _builtins

_readline_available = False

try:
    import readline
    _readline_available = True
except ImportError:
    pass

class Completer:
    def __init__(self, namespace: dict):
        self.namespace = namespace
        self.matches = []

    def complete(self, text: str, state: int) -> str:
        if not _readline_available:
            return '\t'

        namespace = self.namespace

        if not text.strip():
            if state == 0:
                readline.insert_text('\t')
                readline.redisplay()
                return ''
            else:
                return None

        if state == 0:
            if '.' in text:
                self.matches = self.match_attr(text)
            else:
                self.matches = self.match_name(text)

        try:
            return self.matches[state]
        except IndexError:
            return None

    def __check_callable_prefix(self, word, obj) -> str:
        return ('%s(' % word) if callable(obj) else word

    def match_name(self, text: str):
        from ..core.aparser import _keywords

        ns = self.namespace
        matches = []
        n = len(text)

        for word in _keywords:
            if word not in matches:
                if word[:n] == text:
                    matches.append(word)

        for word in ns.keys():
            if '::' in word:
                continue

            if word[:n] == text:
                try:
                    word = self.__check_callable_prefix(word, ns[word])
                except Exception:
                    pass
                matches.append(word)

        for word in dir(_builtins):
            if '::' in word:
                continue

            if word not in matches:
                if word[:n] == text:
                    try:
                        word = self.__check_callable_prefix(
                                word, getattr(_builtins, word))
                    except Exception:
                        pass
                    matches.append(word)

        return matches

    def match_attr(self, text: str):
        import re
        m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
        if not m:
            return []

        expr, attr = m.group(1, 3)

        try:
            this = eval(expr, self.namespace)
        except Exception:
            return []

        words = set(dir(this))

        n = len(attr)

        matches = []

        show_prefix = attr[:1] == '_'

        for word in words:
            if '::' in word:
                continue

            if word[:1] == '_' and not show_prefix:
                continue
            if word[:n] == attr:
                try:
                    word = self.__check_callable_prefix(word, getattr(this, word))
                except Exception:
                    pass

                matches.append('%s.%s' % (expr, word))

        matches.sort()
        return matches

_AIL_PYC_MODULE_ = True
_AIL_NAMESPACE_ = {'Completer': Completer}


# encoding: utf-8
# pylint: disable=too-few-public-methods
"""
measurement schemas
------------
"""
from marshmallow import fields, Schema, post_load
from .models import Measurement

class MeasurementSchema(Schema):
    msg_id = fields.Int()
    start_time = fields.DateTime()
    device_id = fields.Int()
    mea_id = fields.Int()
    c1 = fields.Float()
    c2 = fields.Float()
    c3 = fields.Float()

    class Meta:
        #load_only = ("msg_id",)
        #fields = ("start_time", "c1", "c2", "c3")
        load_only = ("msg_id", "device_id", "mea_id")
        ordered = True

    @post_load
    def make_Measurement(self, data):
        return Measurement(**data)

class MeaQuerySChema(Schema):
    "This calss used to validate the input parameters for query measurement"
    #dev_id = fields.Int(required=True, allow_none=False)
    dev_id = fields.List(fields.Int, required=True, allow_none=False)
    stime = fields.DateTime(required=False)
    etime = fields.DateTime(required=False)


#Love Babbar: Lecture 4 Solving pattern question
#Print the pattern in this format
# Enter n: 5
#     1
#    121
#   12321
#  1234321
# 123454321

n = int(input("Enter n: "))
row = 1
while(row<=n):
    # Printing space triangle
    space = n - row
    while(space):
        print(" ", end='')
        space = space - 1

    # Printing Second Tirangle
    col=1
     # For Printing Triangle, run the loop till row
    while(col<=row):
        print(col,end='')
        col = col + 1

    # Printing Third Tirangle
    start = row - 1
    while(start):
        print(start, end='')
        start = start - 1
    print()
    row = row + 1

# Output:
# Enter n: 5
#     1
#    121
#   12321
#  1234321
# 123454321


import numpy as np

def unpickle(file):
    import cPickle
    with open(file, 'rb') as fo:
        dict = cPickle.load(fo)
    return dict

def data_cifar10():
	dataset_folder = '../dataset/cifar-10-batches-py/'
	dataset = {}
	for i in range(1,6):
		train_filename = 'data_batch_'+str(i)
		train_data = unpickle(dataset_folder+train_filename)
		if i == 1:
			dataset['training_features'] = train_data['data']
			dataset['training_labels'] = np.array(train_data['labels'])
		else:
			dataset['training_features'] = np.concatenate((dataset['training_features'],train_data['data']))
			dataset['training_labels'] = np.concatenate((dataset['training_labels'],np.array(train_data['labels'])))
	test_filename = 'test_batch'
	test_data = unpickle(dataset_folder+test_filename)
	dataset['test_features'] = test_data['data']
	dataset['test_labels'] = np.array(test_data['labels'])
	dataset['label_names'] = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
	return dataset


import os
import shutil
from .base import GnuRecipe

class Extra:
    pass

class GhostScriptRecipe(GnuRecipe):
    def __init__(self, *args, **kwargs):
        super(GhostScriptRecipe, self).__init__(*args, **kwargs)
        self.sha256 = 'a2dec70ab32ee83a05b8e257dfe3f308' \
                      '2ebe3a3d6c0f4c141b3b50ed4bda1564'

        self.name = 'ghostscript'
        self.version = '9.20'
        self.simple_version = self.version.replace('.', '')
        self.depends = ['expat',
                        'fontconfig',
                        'freetype',
                        'gtk3',
                        'libjpeg-turbo',
                        'libpng',
                        'libtiff',
                        'lcms']
        self.url = 'https://github.com/ArtifexSoftware/ghostpdl-downloads/' \
                   'releases/download/gs%s/' \
                   'ghostpdl-$version.tar.gz' % self.simple_version
        self.configure_args += ['--enable-dynamic',
                                '--disable-compile-inits',
                                '--with-system-libtiff']
        self.compile_args = [self.compile_args,
                             ['make', 'so', '-j%s' % self.cpu_count]]

        self.install_args = [
            self.install_args,
            ['make', 'soinstall'],
            ['install', '-v', '-m644', 'base/*.h',
             '%s/include/ghostscript' % self.prefix_dir],
            ['ln', '-v', '-s', 'ghostscript', '%s/include/ps' % self.prefix_dir]
        ]
        self.fonts = Extra()
        self.fonts.url = 'http://downloads.sourceforge.net/gs-fonts/' \
                         '$name-$version.tar.gz'
        self.fonts.version = '8.11'
        self.fonts.name = 'ghostscript-fonts-std'
        self.fonts.sha256 = '0eb6f356119f2e49b2563210852e17f5' \
                            '7f9dcc5755f350a69a46a0d641a0c401'
        self.extra_downloads.append(self.fonts)

        self.fonts2 = Extra()
        self.fonts2.url = 'http://downloads.sourceforge.net/gs-fonts/' \
                          '$name-$version.tar.gz'
        self.fonts2.version = '6.0'
        self.fonts2.name = 'gnu-gs-fonts-other'
        self.fonts2.sha256 = '8146cccc4699fe9dab841446bdd17039' \
                             'f4769c903eceb54409188b920754aab3'

        self.extra_downloads.append(self.fonts2)

    def install(self):
        super(GhostScriptRecipe, self).install()
        self.log_dir('install', self.directory, 'installing fonts')

        args = ['tar', '-xvf', self.fonts.filename, '-C',
                '%s/share/ghostscript' % self.prefix_dir, '--no-same-owner']
        self.run_exe(args, self.directory, self.environment)

        args = ['tar', '-xvf', self.fonts2.filename, '-C',
                '%s/share/ghostscript' % self.prefix_dir, '--no-same-owner']
        self.run_exe(args, self.directory, self.environment)

        args = ['fc-cache', '-v',
                '%s/share/ghostscript/fonts/' % self.prefix_dir]
        self.run_exe(args, self.directory, self.environment)

#    def patch(self):
#        self.log_dir('patch', self.directory, 'removing bundled libs')
#        shutil.rmtree(os.path.join(self.directory, 'freetype'))
#        shutil.rmtree(os.path.join(self.directory, 'lcms2'))
#        shutil.rmtree(os.path.join(self.directory, 'jpeg'))
#        shutil.rmtree(os.path.join(self.directory, 'libpng'))
#        shutil.rmtree(os.path.join(self.directory, 'zlib'))

        r'''TODO: (From BLFS)
Optional

Cairo-1.12.16, Cups-1.7.5, Fontconfig-2.11.1 (required, if you are installing any suggested font), GTK+-2.24.24, libidn-1.29, libpaper-1.1.24+nmu3, Little CMS-1.19 (not used by default, nor if lcms2 is present or found), and X Window System

User Notes: http://wiki.linuxfromscratch.org/blfs/wiki/gs

Installation of Ghostscript

[Note] Note
The Ghostscript build system is not user-friendly. In order to use system copies of various graphics libraries, you must do it using unconventional methods.

GPL Ghostscript includes (old) copies of several libraries. Some of these seem to have been patched to fix known vulnerabilities, but others of these copies are less-well maintained. To ensure that any future fixes are applied throughout the whole system, it is recommended that you first install the released versions of these libraries and then configure GPL Ghostscript to link to them.

If you have installed these dependencies on your system, remove the copies of expat, freetype, lcms2, libjpeg, and libpng:

sed -i 's/ZLIBDIR=src/ZLIBDIR=$includedir/' configure.ac configure &&
rm -rf expat freetype lcms2 jpeg libpng
Compile Ghostscript:

rm -rf zlib &&
./configure --prefix=/usr --disable-compile-inits \
 --enable-dynamic --with-system-libtiff &&
make
[Note] Note
The shared library depends on GTK+-2.24.24. It is only used in external programs like ImageMagick-6.8.9-7.

To compile the shared library libgs.so, run the following additional command as an unprivileged user:

make so
This package does not come with a test suite. However, you may test the operation of the newly built gs program by issuing the following command (issue from an X Window System terminal):

bin/gs -Ilib -IResource/Init -dBATCH examples/tiger.eps
Now, as the root user:

make install
If you want the shared library too:

make soinstall &&
install -v -m644 base/*.h /usr/include/ghostscript &&
ln -v -s ghostscript /usr/include/ps
Now make the documentation accessible from the normal place:

ln -sfv ../ghostscript/9.14/doc /usr/share/doc/ghostscript-9.14
If you have downloaded any fonts, unpack them to /usr/share/ghostscript and ensure the ownerships of the files are root: root. Substitute <font-tarball> appropriately in the command below for the fonts you wish to install:

tar -xvf ../<font-tarball> -C /usr/share/ghostscript --no-same-owner &&
fc-cache -v /usr/share/ghostscript/fonts/
'''


from naive.optimize import *
import numpy as np
import time
import inspect

class timer:
    def __enter__(self):
        self.time = time.clock()
        return self
    def __exit__(self, type, value, traceback):
        print(f"[TIME] {time.clock() - self.time:.4f}s")

def program_problem_3_2(n=2, subproblem=1):
    if subproblem == 1:
        def f(x):
            '''
            Waston Function. 2 <= n <= 31.
            '''
            res = 0
            for i in range(1, 30):
                r = -1
                for j in range(2, n+1):
                    r += (j - 1) * x[j-1] * (i/29)**(j-2)
                tmp = 0
                for j in range(1, n+1):
                    tmp += x[j-1] * (i/29)**(j-1)
                r -= tmp**2
                res += r**2
            res += x[0]**2
            res += (x[1] - x[0]**2 - 1)**2
            return res
        x0 = np.zeros(n)

    elif subproblem == 2:
        h = 1/(n+1)
        def t(i):
            return i * h
        def f(x):
            '''
            Discrete Boundary Value Function
            '''
            res = 0
            for i in range(n):
                if i == 0:
                    r = 2 * x[i] - x[i+1] + h**2 * (x[i] + t(i+1) + 1)**3 / 2
                elif i == n-1:
                    r = 2 * x[i] - x[i-1] + h**2 * (x[i] + t(i+1) + 1)**3 / 2
                else:
                    r = 2 * x[i] - x[i-1] - x[i+1] + h**2 * (x[i] + t(i+1) + 1)**3 / 2
                res += r**2
            return res
        x0 = [t(i+1)*(t(i+1)-1) for i in range(n)]

    elif subproblem == 0:
        def f(x):
            '''
            simple test
            '''
            G = np.array([[21,4],[4,15]])
            b = np.array([2,3])
            c = 10
            return x @ G @ x / 2 + b @ x + c
        x0 = [-30, 100]

    for method in [SteepestDescent,
                   Newton,
                   partial(QuasiNewton, method='SR1'),
                   partial(QuasiNewton, method='DFP'),
                   partial(QuasiNewton, method='BFGS'),
                   partial(BB, method='BB1'),
                   partial(BB, method='BB2'),
                   ]:

        print(f"[METHOD] {method.__name__ if hasattr(method, '__name__') else method.func.__name__}, {inspect.signature(method)}")
        with timer():
            x = method(f, x0)
        print(f"[X] x = {x}")
        print(f"[Y] f(x) = {f(x)}")
        print("")

if __name__ == '__main__':
    #program_problem_3_2(10, 1)
    program_problem_3_2(16, 2)


import sys
sys.stdin = open("baek11650.txt")

N = int(input())
MAP = [0] * N
for i in range(0, N):
    MAP[i] = list(map(int, input().split()))
MAP.sort(key=lambda x: (x[0], x[1]))
for i in range(0, N):
    print(MAP[i][0], end=" ")
    print(MAP[i][1])


import numpy as np
from .. import Circuit, Node, DcOp, Resistor, Capacitor, Diode, Isrc, Mos

class CmosLatch(Circuit):
    """ Back-to-Back CMOS Inverter Latch """

    def define(self):
        self.create_nodes(2)
        vdd = self.create_forced_node(name='vdd', v=1.0)

        self.create_comp(cls=Mos, polarity=1,
                         conns={'s': self.node0, 'b': self.node0,
                                'd': self.nodes[1], 'g': self.nodes[0]})
        self.create_comp(cls=Mos, polarity=-1,
                         conns={'s': vdd, 'b': vdd,
                                'd': self.nodes[1], 'g': self.nodes[0]})

        self.create_comp(cls=Mos, polarity=1,
                         conns={'s': self.node0, 'b': self.node0,
                                'd': self.nodes[0], 'g': self.nodes[1]})
        self.create_comp(cls=Mos, polarity=-1,
                         conns={'s': vdd, 'b': vdd,
                                'd': self.nodes[0], 'g': self.nodes[1]})

def sim(x0=None):
    """ Create an instance of the circuit under test, and simulate it. """
    ckt = CmosLatch()
    s = DcOp(ckt=ckt)
    s.solve(x0)
    print(s.v)
    return s

def test_stuff():
    """ Run some sims!
    Sweep and plot a few RC parameters. """
    sim([0.5, 0.5])


# -*- coding: utf-8 -*-
"""
Created on Tue Dec  7 16:14:46 2021

@author: 201921250004
"""
import numpy as np
import random
import networkx as nx
from pick_edges import pick_edges
from pick_edges import random_sample_edges
from RealNetwork import RealNetwork
from itertools import combinations
import operator
class SignedNetwork(RealNetwork):
    def __init__(self,Network):
        self.NetworkType = Network['NetworkType']
        if self.NetworkType in ['er','ws','re','ba']:
            #self.NetworkType=Network["NetworkType"]
            self.N=Network["N"]
            self.k=Network["k"]
            self.p_ws=Network.get("p_ws",None) #小世界网络的重连概率
            self.strategy = Network.get("strategy","random")
            self.p_n = Network.get('p_n',None) #随机撒负边
            #self.p = Network['p']
            #self.q=Network.get("q",None)
            if self.strategy == "random":
                if self.NetworkType=="er":#ER network
                    self.nosignNetwork = nx.erdos_renyi_graph(self.N, self.k/(self.N-1))
                if self.NetworkType=="ws":#ER network
                    self.nosignNetwork = nx.watts_strogatz_graph(self.N,self.k,self.p_ws)
                if self.NetworkType=="ba":#BA network
                    self.nosignNetwork = nx.barabasi_albert_graph(self.N,self.k)

                if self.NetworkType=="re":#Re network
                    self.nosignNetwork = nx.random_regular_graph(self.k,self.N)
            else:
                #小世界网络断边重连
                if self.NetworkType=="ws":
                    self.nosignNetwork = nx.random_regular_graph(self.k,self.N)
                    n_edges_data = list(self.nosignNetwork.edges.data())
                    selected_edges = random_sample_edges(self.nosignNetwork.edges,self.p_ws)
                    for u,v,wt in n_edges_data:
                        if (u,v) in selected_edges:
                            self.nosignNetwork.remove_edge(u, v)
                            if random.random()<0.5:
                                new_node = self.relink(u)
                                self.nosignNetwork.add_edge(u,new_node)
                                self.nosignNetwork[u][new_node]['new_link']=1
                            else:
                                new_node = self.relink(v)
                                self.nosignNetwork.add_edge(v,new_node)
                                self.nosignNetwork[v][new_node]['new_link']=1
                        else:
                            self.nosignNetwork[u][v]['new_link']= 0
                elif self.NetworkType=="ba":
                    self.nosignNetwork = nx.barabasi_albert_graph(self.N,self.k)
                    self.edge_betweenness_centrality =nx.centrality.edge_betweenness_centrality(self.nosignNetwork)
                    selected_edges = sorted(self.edge_betweenness_centrality, key=operator.itemgetter(1),reverse=True)
                    selected_edges = selected_edges[:int(len(selected_edges)*self.p_ws)]
                    for u,v,wt in self.nosignNetwork.edges.data():
                        if (u,v) in selected_edges:
                            self.nosignNetwork[u][v]['new_link']=1
                        else:
                            self.nosignNetwork[u][v]['new_link']=0

            #self.p_po = self.cal_po_edge_p()
            #self.p_ne = 1-self.p_po
            if self.p_n != None:
                self.p = self.cal_p()
                self.q = self.cal_q()
        elif self.NetworkType == 'real':
            self.readtxt(filename="dataset\\"+Network['name'])
    def cal_q(self):
        return self.k*self.p_n/((self.N-1)*(1-self.p))
    def cal_p(self):
        return (1-self.p_n)*self.k/(self.N-1)
    def relink(self,u):
        neighbor_list = list(nx.neighbors(self.nosignNetwork,u))
        node_list = list(self.nosignNetwork.nodes)
        for del_node in neighbor_list:
            node_list.remove(del_node)
        node_list.remove(u)
        return random.sample(node_list,1)[0]
    def add_Sign(self,p_n,alpha1=None):
        self.signedNetwork=self.nosignNetwork
        self.p_n=p_n
        self.p = self.cal_p()
        self.q = self.cal_q()
        if self.strategy == 'random':
            selected_edges = random.sample(self.signedNetwork.edges,int(len(self.signedNetwork.edges)*self.p_n))
        else:
            new_link_edges = pick_edges(self.signedNetwork,{'new_link':1})
            old_link_edges = pick_edges(self.signedNetwork,{'new_link':0})
            selected_edges = random_sample_edges(new_link_edges,self.p_n*alpha1/self.p_ws)
            selected_edges += random_sample_edges(old_link_edges,self.p_n*(1-alpha1)/(1-self.p_ws))
            #selected_edges = random.sample(new_link_edges,int(len(new_link_edges)*self.p_n*alpha1/self.p_ws))
            #selected_edges += random.sample(old_link_edges,int(len(old_link_edges)*self.p_n*(1-alpha1)/(1-self.p_ws)))
        for u,v,wt in self.signedNetwork.edges.data():
                if (u,v) in selected_edges:
                    self.signedNetwork[u][v]["ne_weight"]=int(1)
                    self.signedNetwork[u][v]["po_weight"]=int(0)
                else:
                    self.signedNetwork[u][v]["ne_weight"]=int(0)
                    self.signedNetwork[u][v]["po_weight"]=int(1)
            #for u,v,wt in self.signedNetwork.edges.data():
                #一半分布在长程边，一半分布在非长程边
                #alpha1表示负边在长程边上的比例
                '''
                if self.signedNetwork[u][v]['mark'] == 'new_link':
                    if random.random()<self.p_n*alpha1/self.p_ws:
                        self.signedNetwork[u][v]["ne_weight"]=int(1)
                        self.signedNetwork[u][v]["po_weight"]=int(0)
                    else:
                        self.signedNetwork[u][v]["ne_weight"]=int(0)
                        self.signedNetwork[u][v]["po_weight"]=int(1)
                else:
                    if random.random()<self.p_n*(1-alpha1)/(1-self.p_ws):
                        self.signedNetwork[u][v]["ne_weight"]=int(1)
                        self.signedNetwork[u][v]["po_weight"]=int(0)
                    else:
                        self.signedNetwork[u][v]["ne_weight"]=int(0)
                        self.signedNetwork[u][v]["po_weight"]=int(1)
                '''
        return self

    def get_degree(self):
        self.po_degree_array=np.array(list(self.signedNetwork.degree(weight="po_weight")))
        self.ne_degree_array=np.array(list(self.signedNetwork.degree(weight="ne_weight")))
        self.po_mean=np.mean(self.po_degree_array[:,1])
        self.ne_mean=np.mean(self.ne_degree_array[:,1])
        print("%.2f"%self.po_mean,"%.2f"%self.ne_mean)
        return self.po_mean,self.ne_mean
    def get_delta_degree(self):
        self.po_degree_array=self.signedNetwork.degree(weight="po_weight")
        self.ne_degree_array=self.signedNetwork.degree(weight="ne_weight")
        self.degree_dict = {}
        for node in self.signedNetwork.nodes():
            self.degree_dict[node] = {}
            self.degree_dict[node]["po_degree"]=self.po_degree_array[node]
            self.degree_dict[node]["ne_degree"]=self.ne_degree_array[node]
            po_neighbor_list = list(filter(lambda x: self.signedNetwork[node][x]['po_weight']==1,self.signedNetwork.neighbors(node)))
            num_po_triangles = 0
            num_ne_triangles = 0
            num_triangles = 0
            for node1,node2 in combinations(po_neighbor_list,2):
                num_triangles += 1
                if self.signedNetwork.has_edge(node1,node2):
                    if self.signedNetwork[node1][node2]['po_weight']==1:
                        num_po_triangles += 1
                    elif self.signedNetwork[node1][node2]['ne_weight']==1:
                        num_ne_triangles += 1
            self.degree_dict[node]["num_triangles"]=num_triangles
            self.degree_dict[node]["num_po_triangles"]=num_po_triangles
            self.degree_dict[node]["num_ne_triangles"]=num_ne_triangles
        return 1

'''
Network = {
'N':100,
'k':6,
'p':0.01,
'p_n':0.2,
'NetworkType':'ba'
}
for p_n in np.arange(0,1,0.01):
    sn = SignedNetwork(Network)
    sn.add_Sign(p_n)
    sn.get_degree()
'''


from operator import mod
import pandas as pd
import scipy.stats as stats
from itertools import product
data=pd.read_csv("C:/Users/abomb/Projects/microbiotaPhd/allDiets_collection_total.csv")

def getCombinations(var):
    varList=pd.unique(data[var].tolist())
    output = list(product(varList, varList))
    return output

def defGetDietComb():
    normal=data.loc[data['Type']=='N' ]
    normalList=pd.unique(normal['Diet'].tolist())
    modified=data.loc[(data['Type']=='HF') | (data['Type']=='HS')]
    modifiedList=pd.unique(modified['Diet'].tolist())
    output = list(product(normalList, modifiedList))
    return output

# function to get p values of comparisons statistic and difference in means

def mwTest(compare, variable, treatment):
    comparisons=[]
    pVals=[]
    meanDifference=[]
    groups=defGetDietComb()
    dataSel=data.loc[(data['Treatment']==treatment)]
    for i in groups:
        var1=i[0]
        var2=i[1]
        varComparison=(var1+'_vs_'+var2)
        group1=dataSel.loc[(dataSel[variable]==var1)]
        xVar=group1[compare].dropna()
        xVarMean=xVar.mean()
        group2=dataSel.loc[(dataSel[variable]==var2)]
        yVar=group2[compare].dropna()
        yVarMean=yVar.mean()
        U1, p=stats.mannwhitneyu(xVar, yVar, nan_policy='omit')
        difference=(xVarMean-yVarMean)
        # append lists
        comparisons.append(varComparison)
        pVals.append(p)
        meanDifference.append(difference)
    testResults=pd.DataFrame({'Comparisons': comparisons, 'pVal': pVals, 'Mean_Difference': meanDifference})
    return(testResults)

totalDiet=mwTest(compare='Sum(Total Collected)', variable='Diet', treatment='S')
totalDeitSel=totalDiet.iloc[[1,5,4,8,10,16,20]].reset_index(drop=True)

totalDeitSel.to_csv("C:/Users/abomb/Projects/microbiotaPhd/output/totalDietS.csv", index=False)


import logging
import os
import json

from flask import Flask
from flask_ask import Ask, request, session, question, statement
import subprocess

app = Flask(__name__)
ask = Ask(app, "/")
logging.getLogger('flask_ask').setLevel(logging.DEBUG)

STATUSON = ['on','high']
STATUSOFF = ['off','low']
BASE_COMMAND = "/usr/local/bin/pilight-send -p %s "
UNIT_COMMAND = "%s %d -u %d %s "
NAME_COMMAND = "-n %s "
PROGRAM_COMMAND = "%s %d -u %d "
SERVER_COMMAND = "%s -S=127.0.0.1 -P=5000"

PILIGHT_CONFIG = "/etc/pilight/config.json"

SEND_ON="-t"
SEND_OFF="-f"
SYS_CODE="-s"
ID_CODE="-i"

device_list = []
class Device:
    def __init__(self, name, protocol, id, unit, systemcode):
        self.name = name
        self.protocol = protocol
        self.id = id
        self.unit = unit
        self.systemcode = systemcode

def loadDevices():
    with open(PILIGHT_CONFIG) as json_file:
        data = json.load(json_file)

    node = data['devices']
    for key, value in node.items():
        unit_node = value['id'][0]
        protocol = value['protocol'][0]
        systemcode = False
        if 'id' in unit_node:
            id = unit_node['id']
            unit = unit_node['unit']
        elif protocol is 'program':
            id = unit_node['name']
            unit = 0
            systemcode = False
        else:
            id = unit_node['systemcode']
            unit = unit_node['unitcode']
            systemcode = True

        device_list.append(Device(key, protocol, id, unit, systemcode))

def getDevice(name):
    return next((device for device in device_list if device.name.lower() == name.lower()), None)

#loadDevices()

@ask.launch
def launch():
    speech_text = 'Welcome to Raspberry Pi Automation. Supported devices are: '.join(device_list)
    return question(speech_text).reprompt(speech_text).simple_card(speech_text)

@ask.intent('pilight', mapping = {'status':'status','deviceName':'device'})
def pilight(status,deviceName):
    print(deviceName, status)
    device = getDevice(deviceName)

    if device is not None:
        switch(device, status)
        return statement('turning {} {} lights'.format(status, device.name))
    else:
        return statement('Sorry not possible.')

@ask.intent('AMAZON.HelpIntent')
def help():
    speech_text = 'You can say hello to me!'
    return question(speech_text).reprompt(speech_text).simple_card('HelloWorld', speech_text)

def switch(device, status):
    system_flag = SYS_CODE if device.systemcode else ID_CODE
    on_flag = SEND_ON if status in STATUSON else SEND_OFF

    if device.protocol is not 'program':
        command = COMMAND % (device.protocol) + UNIT_COMMAND % (system_flag, device.id, device.unit) + BASE_COMMAND % (on_flag)
    else:
        command = COMMAND % (device.protocol) + NAME_COMMAND % (device.id) + BASE_COMMAND % on_flag

    print(command)
    returned_value = subprocess.call(command, shell=True)  # returns the exit code in unix
    print('returned value:', returned_value)
    return True

@ask.session_ended
def session_ended():
    return "{}", 200

if __name__ == '__main__':
    if 'ASK_VERIFY_REQUESTS' in os.environ:
        verify = str(os.environ.get('ASK_VERIFY_REQUESTS', '')).lower()
        if verify == 'false':
            app.config['ASK_VERIFY_REQUESTS'] = False
    loadDevices()
    app.run(host='0.0.0.0', port=5003, debug=True)


from Qt import QtCore, QtWidgets
from NodeGraphQt.widgets.stylesheet import *
from NodeGraphQt import BaseNode, NodeBaseWidget

class AddPortWidget(QtWidgets.QWidget):
    """
    Custom widget to be embedded inside a node.
    """

    def __init__(self, parent=None):
        super(AddPortWidget, self).__init__(parent)
        self.combo_1 = QtWidgets.QComboBox()
        self.btn_input = QtWidgets.QPushButton('< Add Input')
        self.btn_output = QtWidgets.QPushButton('Add Output >')

        self.name_edit = QtWidgets.QLineEdit()
        self.name_edit.setText('')
        self.name_edit.setStyleSheet(STYLE_QLINEEDIT)
        self.name_edit.setAlignment(QtCore.Qt.AlignCenter)

        layout = QtWidgets.QVBoxLayout(self)
        layout.setContentsMargins(0, 0, 0, 0)
        layout.addWidget( self.name_edit )
        layout.addWidget( self.btn_input )
        layout.addWidget( self.btn_output )

class AddPortWidgetWrapper(NodeBaseWidget):
    """
    Wrapper that allows the widget to be added in a node object.
    """

    def __init__(self, parent=None):
        super(AddPortWidgetWrapper, self).__init__(parent)

        # set the name for node property.
        self.set_name('my_widget')

        # set the label above the widget.
        self.set_label('Custom Ports')

        # set the custom widget.
        self.set_custom_widget(AddPortWidget())

        # connect up the signals & slots.
        self.wire_signals()

    def wire_signals(self):
        widget = self.get_custom_widget()

        # wire up the button.
        widget.btn_input.clicked.connect(self.on_btn_input_clicked)
        widget.btn_output.clicked.connect(self.on_btn_output_clicked)

    def on_btn_input_clicked(self):
        widget = self.get_custom_widget()
        port_name = widget.name_edit.text()
        if not port_name: return

        try:
            self.node.add_input( port_name )
        except:
            print( "Input with name " + port_name + " already exists!" )

    def on_btn_output_clicked(self):
        widget = self.get_custom_widget()
        port_name = widget.name_edit.text()
        if not port_name: return

        try:
            self.node.add_output( port_name )
        except:
            print( "Output with name " + port_name + " already exists!" )

    def get_value(self):
        pass

    def set_value(self, value):
        pass


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import sys

from tensorflow.examples.tutorials.mnist import input_data

import tensorflow as tf
import numpy as np

FLAGS = None

batch_size = 32

class Discriminator:
    def __init__(self):
        with tf.variable_scope('discriminator'):
            self.W_conv1 = weight_variable([5, 5, 1, 1024])
            self.b_conv1 = bias_variable([1024])

            self.W_conv2 = weight_variable([5, 5, 1024, 1])
            self.b_conv2 = bias_variable([1])

            self.W_fc1 = weight_variable([49, 1024])
            self.b_fc1 = bias_variable([1024])

            self.W_fc2 = weight_variable([1024, 100])
            self.b_fc2 = bias_variable([100])

            self.W_fc3 = weight_variable([100, 1024])
            self.b_fc3 = bias_variable([1024])

            self.W_fc4 = weight_variable([1024, 1])
            self.b_fc4 = bias_variable([1])

    def embed(self, x):
        x = tf.reshape(x, [-1, 28, 28, 1])

        x = tf.nn.conv2d(x, self.W_conv1, [1, 2, 2, 1], "SAME") + self.b_conv1
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.nn.conv2d(x, self.W_conv2, [1, 2, 2, 1], "SAME") + self.b_conv2
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.reshape(x, [-1, 49])

        x = tf.matmul(x, self.W_fc1) + self.b_fc1
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.matmul(x, self.W_fc2) + self.b_fc2
        x = tf.nn.sigmoid(x)

        return x

    def discriminate(self, x):
        x = self.embed(x)

        x = tf.matmul(x, self.W_fc3) + self.b_fc3
        x = tf.nn.tanh(x)

        x = tf.matmul(x, self.W_fc4) + self.b_fc4
        x = tf.nn.sigmoid(x)

        return x

class Generator:
    def __init__(self):
        with tf.variable_scope('generator'):
            self.W_fc1 = weight_variable([100, 100])
            self.b_fc1 = bias_variable([100])

            self.W_fc2 = weight_variable([100, 7*7*1024])
            self.b_fc2 = bias_variable([7*7*1024])

            self.W_dconv0 = weight_variable([5, 5, 64, 1024])
            self.b_dconv0 = bias_variable([64])

            self.W_dconv1 = weight_variable([5, 5, 16, 64])
            self.b_dconv1 = bias_variable([16])

            self.W_dconv2 = weight_variable([5, 5, 1, 16])
            self.b_dconv2 = bias_variable([1])

    def generate(self, x):
        x = tf.matmul(x, self.W_fc1) + self.b_fc1
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.matmul(x, self.W_fc2) + self.b_fc2
        x = tf.reshape(x, [-1, 7, 7, 1024])
        # input = tf.nn.batch_normalization(input, 0, 0.1, 0, 1, 1e-5)
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.nn.conv2d_transpose(x, self.W_dconv0, [int(x.get_shape()[0]), 14, 14, 64], [1, 2, 2, 1], "SAME") + self.b_dconv0
        # h_dconv0 = tf.nn.batch_normalization(h_dconv0, 0, 0.1, 0, 1, 1e-5)
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.nn.conv2d_transpose(x, self.W_dconv1, [int(x.get_shape()[0]), 28, 28, 16], [1, 2, 2, 1], "SAME") + self.b_dconv1
        # h_dconv1 = tf.nn.batch_normalization(h_dconv1, 0, 0.1, 0, 1, 1e-5)
        # x = tf.nn.relu(x)
        x = tf.nn.tanh(x)

        x = tf.nn.conv2d_transpose(x, self.W_dconv2, [int(x.get_shape()[0]), 28, 28, 1], [1, 1, 1, 1], "SAME") + self.b_dconv2
        x = tf.nn.sigmoid(x)

        x = tf.reshape(x, (-1, 28*28))
        return x

def weight_variable(shape):
    """weight_variable generates a weight variable of a given shape."""
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    """bias_variable generates a bias variable of a given shape."""
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def fake_embedding(n):
    return np.asarray(np.random.random_sample((n, 100)), np.float32)

def main(_):
    mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

    discriminator = Discriminator()
    generator = Generator()

    rx = tf.placeholder(tf.float32, [batch_size, 784])
    ry = discriminator.discriminate(rx)
    re = discriminator.embed(rx)
    ro = generator.generate(re)

    ge = tf.placeholder(tf.float32, [batch_size, 100])
    go = generator.generate(ge)
    gy = discriminator.discriminate(go)

    error1 = \
        tf.reduce_mean(tf.square(ry - 1)) + \
        tf.reduce_mean(tf.square(gy))

    error2 = \
        tf.reduce_mean(tf.square(gy - 1))

    error3 = \
        tf.reduce_mean(tf.square(rx - ro))

    error1 = error2 = error3

    learning_rate = tf.placeholder(tf.float32)
    train_step1 = tf.train.GradientDescentOptimizer(learning_rate).\
        minimize(error1, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "discriminator"))
    train_step2 = tf.train.GradientDescentOptimizer(learning_rate).\
        minimize(error2, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator"))
    train_step3 = tf.train.GradientDescentOptimizer(learning_rate).\
        minimize(error3, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator") + tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "discriminator"))
    # train_step1 = tf.train.AdamOptimizer(1e-4).minimize(error1)
    # train_step2 = tf.train.AdamOptimizer(1e-4).minimize(error2)

    with tf.Session() as sess:
        rate = 1e1
        sess.run(tf.global_variables_initializer())
        gene_batch = fake_embedding(batch_size)
        for i in range(20000):
        # for i in range(10):
            batch = mnist.train.next_batch(batch_size)
            real_batch = batch[0]
            if i % 10 == 0:
                rate /= 1.001
            if i % 1 == 0:
                train_error1, _ = \
                    sess.run([error1, train_step1],
                                     feed_dict={
                                         learning_rate: rate,
                                         rx: real_batch, ge: gene_batch
                                     })
                train_error2, _ = \
                    sess.run([error2, train_step2],
                                     feed_dict={
                                         learning_rate: rate,
                                         rx: real_batch, ge: gene_batch
                                     })
                train_error2, _ = \
                    sess.run([error2, train_step2],
                                     feed_dict={
                                         learning_rate: rate,
                                         rx: real_batch, ge: gene_batch
                                     })
                train_error3, _ = \
                    sess.run([error3, train_step3],
                                     feed_dict={
                                         learning_rate: rate,
                                         rx: real_batch, ge: gene_batch
                                     })
                print('step %d, d_loss: %g, g_loss: %g, s_loss: %g'
                      % (i, train_error1, train_error2, train_error3))

            if i % 10 == 0:
                for i in range(10):
                    image = real_batch[i] * 256
                    png = sess.run(tf.image.encode_png(np.reshape(image, [28, 28, 1])))
                    with open("%dr.png" % i, "wb") as f:
                        f.write(png)
                # generated = fake_embedding(100)
                # decoded = sess.run(tf.cast(generator.generate(generated) * 256, tf.uint8))
                _re, decoded = sess.run([re, ro * 256], feed_dict={rx: real_batch})
                for i in range(10):
                    image = decoded[i]
                    png = sess.run(tf.image.encode_png(tf.cast(tf.reshape(image, [28, 28, 1]), tf.uint8)))
                    with open("%d.png" % i, "wb") as f:
                        f.write(png)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str,
                                            default='./mnist/input_data',
                                            help='Directory for storing input data')
    FLAGS, unparsed = parser.parse_known_args()
    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


# coding=utf-8
import glob
import os

from PyQt5.QtCore import QProcess

from cmd import Cmd, CmdType
from file import getFileIconName
from plugin_common.baseplugin import RetVal

__author__ = 'peter'

class FileManagerCmd(Cmd):
    def __init__(self):
        super().__init__()
        self.type = CmdType.fileManager
        self.path = None

    def getDesc(self):
        return self.keyword

    def exec(self, cmd):
        # QProcess.startDetached('xdg-open', [self.path])
        return RetVal.cmd + self.complement(cmd)

    def set(self, name=None, keyword=None, path=None, iconName=None):
        self.name = name
        if keyword:
            self.keyword = keyword
        else:
            self.keyword = self.name
        self.path = path
        self.iconName = iconName
        return self

    def complement(self, cmd):
        if os.path.isdir(self.path):
            return self.keyword + '/'
        else:
            return self.keyword

class FileManager:
    basePath = '/'
    curPath = ''
    showHideFiles = False

    @staticmethod
    def getHomePath():
        return os.path.expanduser('~') + '/'

    @staticmethod
    def setBasePath(path):
        if path == '/':
            FileManager.basePath = '/'
        elif path == '~':
            FileManager.basePath = os.path.expanduser('~') + '/'

    @staticmethod
    def either(c):
        return '[%s%s]*' % (c.lower(), c.upper()) if c.isalpha() else c

    @staticmethod
    def collapsePath(path):
        if path.startswith(FileManager.getHomePath()):
            return path.replace(FileManager.getHomePath(), '~/', 1)
        else:
            return path

    @staticmethod
    def expandPath(path):
        return os.path.expanduser(path)

    @staticmethod
    def getFileList(keyword):
        keyword = ''.join(map(FileManager.either, keyword))
        print(keyword)
        keyword = os.path.expanduser(keyword)
        print(keyword)
        if keyword.endswith('/'):
            return glob.glob(keyword + '*')
        elif keyword.endswith('?') or keyword.endswith('*'):
            return glob.glob(keyword)
        else:
            return glob.glob(keyword + '*')

    @staticmethod
    def Search(keyword):
        ret = []
        for f in sorted(FileManager.getFileList(keyword)):
            ret.append(FileManagerCmd().set(name=os.path.basename(f), path=f,
                                            keyword=FileManager.collapsePath(f),
                                            iconName=getFileIconName(f)))
        return ret