import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import argparse
import os
import joblib

parser = argparse.ArgumentParser()
parser.add_argument("--imgst_dir",default='mimgst',type=str)
parser.add_argument("--dataset",default='voc',choices=['voc','coco','kitti'],type=str,help='dataset name')
parser.add_argument("--mode",default='clip',choices=['mask','clip'],type=str,help='type of secure PatchGuard aggregation; robust masking or clipping')
parser.add_argument("--w",default=8,type=int)
parser.add_argument("--m",default=-1,type=int)
parser.add_argument("--t",default=32,type=float)
parser.add_argument("--eps",default=3,type=int,help='DBSCAN eps')
parser.add_argument("--ms",default=24,type=int,help='DBSCAN min number of samples')

args = parser.parse_args()

DATASET = args.dataset
font = {'family' : 'serif',
        #'weight' : 'bold',
        'size'   : 12}

matplotlib.rc('font', **font)

########################################GT################################
SETTING = '{}_{}_w{}m{}t{:.1f}_{}_{}'.format('gt',args.mode,args.w,args.m,args.t,args.eps,args.ms)
IMGST_DIR = os.path.join(args.imgst_dir,DATASET,SETTING)

prec = np.mean(joblib.load(os.path.join(IMGST_DIR,'prec.z')))
rec = np.mean(joblib.load(os.path.join(IMGST_DIR,'rec.z')))
gt_fa_list = joblib.load(os.path.join(IMGST_DIR,'fa_list.z'))

plt.plot([100],[100],'o',label='Precision-PCD-vanilla',markersize=6)

plt.plot([rec*100],[prec*100],'x',label='Precision-PCD-defended',markersize=6)
gt_rec = rec
###############################################################################

def process(prec,rec,fa_list,conf_cut):
	thres_list=np.linspace(0,0.999,1000)[::-1]
	tmp = thres_list <  conf_cut
	thres_list = thres_list[tmp]
	prec = prec[tmp]
	rec = rec[tmp]
	#prec = np.nanmean(np.where(prec!=0,prec,np.nan),1)
	#rec = np.nanmean(np.where(rec!=0,rec,np.nan),1)
	prec = np.mean(prec,1)
	rec = np.mean(rec,1)
	fa_list = fa_list[tmp]
	tmp = np.argsort(rec)
	rec= rec[tmp]
	prec= prec[tmp]
	fa_list= fa_list[tmp]
	return prec,rec,fa_list
lw = 2

if DATASET == 'voc':
	yolo_cut = 0.945
	frcnn_cut = 0.999
elif DATASET =='coco':
	yolo_cut = 0.9
	frcnn_cut = 0.997
elif DATASET == 'kitti':
	yolo_cut = 0.925
	frcnn_cut =  0.9999
######################YOLO########################################################
prec = joblib.load(os.path.join(args.imgst_dir,args.dataset,'prec_vanilla_{}.z'.format('yolo')))
rec = joblib.load(os.path.join(args.imgst_dir,args.dataset,'rec_vanilla_{}.z'.format('yolo')))
fa_list = joblib.load(os.path.join(args.imgst_dir,args.dataset,'fa_list_vanilla_{}.z'.format('yolo')))

prec,rec,fa_list = process(prec,rec,fa_list,yolo_cut)
plt.plot(rec*100,prec*100,label='Precision-YOLO-vanilla',linestyle='-',linewidth=lw)

#####################YOLO_DPG########################################################
SETTING = '{}_{}_w{}m{}t{:.1f}_{}_{}'.format('yolo',args.mode,args.w,args.m,args.t,args.eps,args.ms)
IMGST_DIR = os.path.join(args.imgst_dir,DATASET,SETTING)

prec = joblib.load(os.path.join(IMGST_DIR,'prec.z'))
rec = joblib.load(os.path.join(IMGST_DIR,'rec.z'))
fa_list = joblib.load(os.path.join(IMGST_DIR,'fa_list.z'))

prec,rec,fa_list = process(prec,rec,fa_list,yolo_cut)
plt.plot(rec*100,prec*100,label='Precision-YOLO-defended',linestyle='-.',linewidth=lw)

yolo_fa_list= fa_list.copy()
yolo_rec = rec.copy()

######################FRCNN########################################################
prec = joblib.load(os.path.join(args.imgst_dir,args.dataset,'prec_vanilla_{}.z'.format('frcnn')))
rec = joblib.load(os.path.join(args.imgst_dir,args.dataset,'rec_vanilla_{}.z'.format('frcnn')))
fa_list = joblib.load(os.path.join(args.imgst_dir,args.dataset,'fa_list_vanilla_{}.z'.format('frcnn')))

prec,rec,fa_list = process(prec,rec,fa_list,frcnn_cut)
plt.plot(rec*100,prec*100,label='Precision-FRCNN-vanilla',linestyle='--',linewidth=lw)

############FRCNN_DPG###############################################

SETTING = '{}_{}_w{}m{}t{:.1f}_{}_{}'.format('frcnn',args.mode,args.w,args.m,args.t,args.eps,args.ms)
IMGST_DIR = os.path.join(args.imgst_dir,DATASET,SETTING)

prec = joblib.load(os.path.join(IMGST_DIR,'prec.z'))
rec = joblib.load(os.path.join(IMGST_DIR,'rec.z'))
fa_list = joblib.load(os.path.join(IMGST_DIR,'fa_list.z'))

prec,rec,fa_list = process(prec,rec,fa_list,frcnn_cut)
plt.plot(rec*100,prec*100,label='Precision-FRCNN-defended',linestyle='-.',linewidth=lw)

frcnn_fa_list= fa_list.copy()
frcnn_rec = rec.copy()

############FAR##########################################################

plt.plot([gt_rec*100],[gt_fa_list*100],'D',label='FAR-PCD-defended',markersize=6)

plt.plot(yolo_rec*100,yolo_fa_list*100,label='FAR-YOLO-defended',linestyle='-',linewidth=lw)
plt.plot(frcnn_rec*100,frcnn_fa_list*100,label='FAR-FRCNN-defended',linestyle='-.',linewidth=lw)

plt.legend(loc='center left')
plt.grid()
plt.xlabel('Recall (%)')
plt.ylabel('Precision / FAR (%)')

plt.xticks(np.arange(10,110,10))
plt.yticks(np.arange(0,110,10))

#plt.xlim([0,1])
#plt.ylim([0,1])
plt.tight_layout()
plt.savefig('clean_{}_{}.png'.format(DATASET,args.mode))
plt.savefig('clean_{}_{}.pdf'.format(DATASET,args.mode))
plt.close()


#!/usr/bin/env python3

import matplotlib.pyplot as plt
import numpy as np

def sigmoid(values):
    return 1.0 / (1.0 + np.exp(-values))

skills = np.linspace(-3.0, 3.0, num=10001)
skills = 0.5 * (skills[1:] + skills[:-1])

difficulties = skills

probabilities = sigmoid(skills[:, None] - difficulties[None, :])
correct = np.random.rand(*probabilities.shape) < probabilities

correct_ratio = np.mean(correct, axis=0)

plt.plot(difficulties, correct_ratio)
# plt.plot(difficulties, sigmoid(-difficulties))
plt.show()

result_diff = []
for i in range(101):
    idx = np.argmin(np.abs(i / 100.0 - correct_ratio))
    result_diff.append(difficulties[idx])
print(result_diff)


import requests
import json
from flask import Flask, render_template, session, request, redirect, url_for, flash
import forms
from flask_session import Session
from flask_bootstrap import Bootstrap4
import msal
import app_config
import lorem
from werkzeug.middleware.proxy_fix import ProxyFix
from datetime import datetime

# initialization
app = Flask(__name__)
app.config.from_object(app_config)
app.config['SECRET_KEY'] = app_config.SECRET_KEY
Session(app)
bootstrap = Bootstrap4(app)
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)

@app.route("/")
def index():
    if not session.get("user"):
        return redirect(url_for("login"))
    return render_template('index.html', user=session["user"], version=msal.__version__)

@app.route("/login")
def login():
    # Technically we could use empty list [] as scopes to do just sign in,
    # here we choose to also collect end user consent upfront
    session["flow"] = _build_auth_code_flow(scopes=app_config.SCOPE)
    return render_template("login.html", auth_url=session["flow"]["auth_uri"], version=msal.__version__)

@app.route(app_config.REDIRECT_PATH)  # Its absolute URL must match your app's redirect_uri set in AAD
def authorized():
    try:
        cache = _load_cache()
        result = _build_msal_app(cache=cache).acquire_token_by_auth_code_flow(
            session.get("flow", {}), request.args)
        if "error" in result:
            return render_template("auth_error.html", result=result)
        session["user"] = result.get("id_token_claims")
        _save_cache(cache)
    except ValueError:  # Usually caused by CSRF
        pass  # Simply ignore them
    return redirect(url_for("index"))

@app.route("/logout")
def logout():
    session.clear()  # Wipe out user and its token cache from session
    return redirect(  # Also logout from your tenant's web session
        app_config.AUTHORITY + "/oauth2/v2.0/logout?post_logout_redirect_uri=" + url_for("index", _external=True))

@app.route("/graphcall")
def graphcall():
    token = _get_token_from_cache(app_config.SCOPE)
    if not token:
        return redirect(url_for("login"))
    graph_data = requests.get(  # Use token to call downstream service
        app_config.ENDPOINT,
        headers={'Authorization': 'Bearer ' + token['access_token']},
    ).json()
    return render_template('display.html', result=graph_data, version=msal.__version__)

@app.route("/get-access-token")
def get_access_token():
    token = _get_token_from_cache(app_config.SCOPE)
    if not token:
        return redirect(url_for("login"))
    token_type = token['token_type']
    access_token = token['access_token']
    expires_in = token['expires_in']
    return render_template('get_access_token.html', raw_access_token=token, token_type=token_type,
                           access_token=access_token, expires_in=expires_in, version=msal.__version__)

@app.route("/send-mail", methods=['GET', 'POST'])
def send_mail():
    if not session.get("user"):
        return redirect(url_for("login"))

    form = forms.SendMailForm()

    # get from email address (userPrincipalName)
    from_email = _get_user_profile_json()['userPrincipalName']

    if form.validate_on_submit():
        recipient = form.recipient.data
        subject = form.subject.data
        content = form.content.data
        save_to_sent_items = form.save_to_sent_items.data

        # get 'access_token'
        token = _get_token_from_cache(app_config.SCOPE)
        if not token:
            return redirect(url_for("login"))
        access_token = token['access_token']

        # send email
        flag = _send_mail(recipient=recipient, subject=subject, content=content, access_token=access_token,
                          save_to_sent_items=save_to_sent_items)
        if flag:
            flash('Your message has been successfully submitted to the corresponding API.')
        else:
            flash('There is an error that occurred. Please refer to the debug log for more information.')

    # define form default values
    form.subject.data = 'TEST MESSAGE [' + str(datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + ']'
    form.content.data = lorem.paragraph()

    return render_template('send_mail.html', form=form, from_email=from_email, version=msal.__version__)

def _send_mail(recipient, subject, content, access_token, save_to_sent_items=False):
    if recipient is not None and subject is not None and content is not None and access_token is not None:
        user_ms_id = _get_user_profile_json()['id']
        url = 'https://graph.microsoft.com/v1.0/users/' + str(user_ms_id) + '/sendMail'
        print(url)
        payload = json.dumps({
            "message": {
                "subject": str(subject),
                "body": {
                    "contentType": "HTML",
                    "content": str(content)
                },
                "toRecipients": [
                    {
                        "emailAddress": {
                            "address": str(recipient)
                        }
                    }
                ]
            },
            "saveToSentItems": str(save_to_sent_items).lower()
        })

        headers = {
            'Authorization': 'Bearer ' + str(access_token),
            'Content-Type': 'application/json'
        }

        # send request
        response = requests.request("POST", url, headers=headers, data=payload)

        # capture response and return to the caller
        print(f"Status Code: {response.status_code}, Response: {response.text}")
        if str(response.status_code) != '202':
            return False
        return True

@app.route('/get-user-profile')
def get_user_profile():
    url = "https://graph.microsoft.com/v1.0/me/"
    payload = {}
    token = _get_token_from_cache(app_config.SCOPE)
    if not token:
        return redirect(url_for("login"))
    access_token = token['access_token']
    headers = {
        'Authorization': 'Bearer ' + str(access_token)
    }
    response = requests.request("GET", url, headers=headers, data=payload).json()
    return render_template('get_user_profile.html', user_profile=response)

def _get_user_profile_json():
    url = "https://graph.microsoft.com/v1.0/me/"
    token = _get_token_from_cache(app_config.SCOPE)
    if not token:
        return redirect(url_for("login"))
    access_token = token['access_token']
    response = requests.request("GET", url, headers={'Authorization': 'Bearer ' + str(access_token)}).json()
    return response

def _load_cache():
    cache = msal.SerializableTokenCache()
    if session.get("token_cache"):
        cache.deserialize(session["token_cache"])
    return cache

def _save_cache(cache):
    if cache.has_state_changed:
        session["token_cache"] = cache.serialize()

def _build_msal_app(cache=None, authority=None):
    return msal.ConfidentialClientApplication(
        app_config.CLIENT_ID, authority=authority or app_config.AUTHORITY,
        client_credential=app_config.CLIENT_SECRET, token_cache=cache)

def _build_auth_code_flow(authority=None, scopes=None):
    return _build_msal_app(authority=authority).initiate_auth_code_flow(
        scopes or [],
        redirect_uri=url_for("authorized", _external=True))

def _get_token_from_cache(scope=None):
    cache = _load_cache()  # This web app maintains one cache per session
    cca = _build_msal_app(cache=cache)
    accounts = cca.get_accounts()
    if accounts:  # So all account(s) belong to the current signed-in user
        result = cca.acquire_token_silent(scope, account=accounts[0])
        _save_cache(cache)
        return result

app.jinja_env.globals.update(_build_auth_code_flow=_build_auth_code_flow)  # Used in template

if __name__ == "__main__":
    app.run()


class Solution:
    def onesMinusZeros(self, grid: List[List[int]]) -> List[List[int]]:
        row=len(grid)
        col=len(grid[0])
        r=[]
        c=[]
        for i in range(row):
            c1=0
            c2=0
            for j in range(col):
                if grid[i][j]==1:
                    c1+=1
                else:
                    c2 +=1
            r.append([c1,c2])
        for j in range(col):
            c1=0
            c2=0
            for i in range(row):
                if grid[i][j]==1:
                    c1 +=1
                else:
                    c2 +=1
            c.append([c1,c2])
        diff=[[0 ]*col for value in range(row)]
        for i in range(len(diff)):
            for j in range(len(diff[0])):
                diff[i][j]=r[i][0]+c[j][0]-r[i][1]-c[j][1]
        return diff


from webcamsettings import WebCamSettings

import pyvirtualcam
import cv2 as cv2
import threading

class VirtualWebcam():
    def __init__(self, webCamSettings: WebCamSettings):
        self.__webCamSettings = webCamSettings

        self.__cam = None
        self.__stopAllThreads = False
        self.__webcamThread = threading.Thread(target=self.__StartVirtualWebcamThread)

    def ConnectToCamera(self, cameraIndex: int, resolution: tuple):
        self.Stop()

        self.__cam = cv2.VideoCapture(cameraIndex, cv2.CAP_DSHOW)
        if not self.__cam.isOpened():
            return False, f"Camera index {cameraIndex} is invalid."

        (width, height) = resolution
        self.__cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
        self.__cam.set(cv2.CAP_PROP_FRAME_HEIGHT, height)

        actualW = self.__cam.get(cv2.CAP_PROP_FRAME_WIDTH)
        actualH = self.__cam.get(cv2.CAP_PROP_FRAME_HEIGHT)

        if actualW == width and actualH == height:
            self.__resolutionWidth = width
            self.__resolutionHeight = height
            self.__stopAllThreads = False
            self.__webcamThread.start()
            return True, ""

        self.__cam.release()
        return False, f"The camera index {cameraIndex} does not support the selected resolution ({width} x {height})"

    def Stop(self):
        if self.__webcamThread.is_alive():
            self.__stopAllThreads = True
            self.__webcamThread.join()

        if self.__cam != None:
            self.__cam.release()

        cv2.destroyAllWindows()

    def __StartVirtualWebcamThread(self):
        with pyvirtualcam.Camera(width=self.__resolutionWidth, height=self.__resolutionHeight, fps=30) as virtualCam:
            self.__webCamSettings.SetVirtualCameraName(f'Using virtual camera: {virtualCam.device}')
            while not self.__stopAllThreads:
                # Capture frame-by-frame
                ret, frame = self.__cam.read()
                # if frame is read correctly ret is True
                if not ret:
                    print("Can't receive frame (stream end?). Exiting ...")
                    break

                frame = self.__ProcessFrame(frame=frame)

                virtualCam.send(frame)
                virtualCam.sleep_until_next_frame()

    def __ProcessFrame(self, frame):
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

        # 1. Apply brightness & contrast
        alpha = self.__webCamSettings.GetContrast() # Contrast control
        beta = self.__webCamSettings.GetBrightness() # Brightness control
        frame = cv2.convertScaleAbs(frame, alpha=alpha, beta=beta)

        # 2. Apply image flip
        hFlip, vFlip = self.__webCamSettings.GetFlip()
        if hFlip:
            frame = cv2.flip(frame, 1)
        if vFlip:
            frame = cv2.flip(frame, 0)
        return frame


import numpy as np

from .._fiff.constants import FIFF
from .._fiff.pick import pick_info
from ..baseline import _log_rescale, rescale
from ..epochs import Epochs
from ..event import make_fixed_length_events
from ..evoked import EvokedArray
from ..fixes import _safe_svd
from ..label import BiHemiLabel, Label
from ..parallel import parallel_func
from ..source_estimate import _make_stc
from ..time_frequency.multitaper import (
    _compute_mt_params,
    _mt_spectra,
    _psd_from_mt,
    _psd_from_mt_adaptive,
)
from ..time_frequency.tfr import cwt, morlet
from ..utils import ProgressBar, _check_option, _pl, _validate_type, logger, verbose
from .inverse import (
    INVERSE_METHODS,
    _assemble_kernel,
    _check_or_prepare,
    _check_ori,
    _pick_channels_inverse_operator,
    _subject_from_inverse,
    combine_xyz,
)

def _restrict_K_to_lbls(labels, K, noise_norm, vertno, pick_ori):
    """Use labels to choose desired sources in the kernel."""
    verts_to_use = [[], []]
    # create mask for K by compiling original vertices from vertno in labels
    for ii in range(len(labels)):
        lab = labels[ii]
        # handle BiHemi labels; ok so long as no overlap w/ single hemi labels
        if lab.hemi == "both":
            l_verts = np.intersect1d(vertno[0], lab.lh.vertices)
            r_verts = np.intersect1d(vertno[1], lab.rh.vertices)  # output sorted
            verts_to_use[0] += list(l_verts)
            verts_to_use[1] += list(r_verts)
        else:
            hidx = 0 if lab.hemi == "lh" else 1
            verts = np.intersect1d(vertno[hidx], lab.vertices)
            verts_to_use[hidx] += list(verts)

    # check that we don't have overlapping vertices in our labels
    for ii in range(2):
        if len(np.unique(verts_to_use[ii])) != len(verts_to_use[ii]):
            raise RuntimeError(
                "Labels cannot have overlapping vertices. "
                "Please select labels with unique vertices "
                "and try again."
            )

    # turn original vertex numbers from vertno into indices for K
    K_mask = np.searchsorted(vertno[0], verts_to_use[0])
    r_kmask = np.searchsorted(vertno[1], verts_to_use[1]) + len(vertno[0])
    K_mask = np.hstack((K_mask, r_kmask))

    # record which original vertices are at each index in out_K
    hemis = ("lh", "rh")
    ki_keys = [
        (hemis[hi], verts_to_use[hi][ii])
        for hi in range(2)
        for ii in range(len(verts_to_use[hi]))
    ]
    ki_vals = list(range(len(K_mask)))
    k_idxs = dict(zip(ki_keys, ki_vals))

    # mask K, handling the orientation issue
    len_allverts = len(vertno[0]) + len(vertno[1])
    if len(K) == len_allverts:
        assert pick_ori == "normal"
        out_K = K[K_mask]
    else:
        # here, K = [x0, y0, z0, x1, y1, z1 ...]
        # we need to drop x, y and z of unused vertices
        assert not pick_ori == "normal", pick_ori
        assert len(K) == 3 * len_allverts, (len(K), len_allverts)
        out_len = len(K_mask) * 3
        out_K = K[0:out_len]  # get the correct-shaped array
        for di in range(3):
            K_pick = K[di::3]
            out_K[di::3] = K_pick[K_mask]  # set correct values for out

    out_vertno = verts_to_use
    if noise_norm is not None:
        out_nn = noise_norm[K_mask]
    else:
        out_nn = None

    return out_K, out_nn, out_vertno, k_idxs

def _prepare_source_params(
    inst,
    inverse_operator,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    pca=True,
    pick_ori="normal",
    prepared=False,
    method_params=None,
    use_cps=True,
):
    """Prepare inverse operator and params for spectral / TFR analysis."""
    inv = _check_or_prepare(
        inverse_operator, nave, lambda2, method, method_params, prepared
    )

    #
    #   Pick the correct channels from the data
    #
    sel = _pick_channels_inverse_operator(inst.ch_names, inv)
    logger.info("Picked %d channels from the data" % len(sel))
    logger.info("Computing inverse...")
    #
    #   Simple matrix multiplication followed by combination of the
    #   three current components
    #
    #   This does all the data transformations to compute the weights for the
    #   eigenleads
    #
    # K shape: (3 x n_sources, n_channels) or (n_sources, n_channels)
    # noise_norm shape: (n_sources, 1)
    # vertno: [lh_verts, rh_verts]

    k_idxs = None
    if not isinstance(label, (Label, BiHemiLabel)):
        whole_K, whole_noise_norm, whole_vertno, _ = _assemble_kernel(
            inv, None, method, pick_ori, use_cps=use_cps
        )
        if isinstance(label, list):
            K, noise_norm, vertno, k_idxs = _restrict_K_to_lbls(
                label, whole_K, whole_noise_norm, whole_vertno, pick_ori
            )
        else:
            assert not label
            K, noise_norm, vertno = whole_K, whole_noise_norm, whole_vertno
    elif isinstance(label, (Label, BiHemiLabel)):
        K, noise_norm, vertno, _ = _assemble_kernel(
            inv, label, method, pick_ori, use_cps=use_cps
        )

    if pca:
        U, s, Vh = _safe_svd(K, full_matrices=False)
        rank = np.sum(s > 1e-8 * s[0])
        K = s[:rank] * U[:, :rank]
        Vh = Vh[:rank]
        logger.info("Reducing data rank %d -> %d" % (len(s), rank))
    else:
        Vh = None
    is_free_ori = inverse_operator["source_ori"] == FIFF.FIFFV_MNE_FREE_ORI

    return K, sel, Vh, vertno, is_free_ori, noise_norm, k_idxs

@verbose
def source_band_induced_power(
    epochs,
    inverse_operator,
    bands,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    n_cycles=5,
    df=1,
    use_fft=False,
    decim=1,
    baseline=None,
    baseline_mode="logratio",
    pca=True,
    n_jobs=None,
    prepared=False,
    method_params=None,
    use_cps=True,
    *,
    verbose=None,
):
    """Compute source space induced power in given frequency bands.

    Parameters
    ----------
    epochs : instance of Epochs
        The epochs.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    bands : dict
        Example : bands = dict(alpha=[8, 9]).
    label : Label | list of Label
        Restricts the source estimates to a given label or list of labels. If
        labels are provided in a list, power will be averaged over vertices.
    lambda2 : float
        The regularization parameter of the minimum norm.
    method : "MNE" | "dSPM" | "sLORETA" | "eLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    n_cycles : float | array of float
        Number of cycles. Fixed number or one per frequency.
    df : float
        Delta frequency within bands.
    use_fft : bool
        Do convolutions in time or frequency domain with FFT.
    decim : int
        Temporal decimation factor.
    baseline : None (default) or tuple, shape (2,)
        The time interval to apply baseline correction. If None do not apply
        it. If baseline is (a, b) the interval is between "a (s)" and "b (s)".
        If a is None the beginning of the data is used and if b is None then b
        is set to the end of the interval. If baseline is equal to (None, None)
        all the time interval is used.
    baseline_mode : 'mean' | 'ratio' | 'logratio' | 'percent' | 'zscore' | 'zlogratio'
        Perform baseline correction by

        - subtracting the mean of baseline values ('mean')
        - dividing by the mean of baseline values ('ratio')
        - dividing by the mean of baseline values and taking the log
          ('logratio')
        - subtracting the mean of baseline values followed by dividing by
          the mean of baseline values ('percent')
        - subtracting the mean of baseline values and dividing by the
          standard deviation of baseline values ('zscore')
        - dividing by the mean of baseline values, taking the log, and
          dividing by the standard deviation of log baseline values
          ('zlogratio')

    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    %(n_jobs)s
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.

        .. versionadded:: 0.16
    %(use_cps_restricted)s

        .. versionadded:: 0.20
    %(verbose)s

    Returns
    -------
    stcs : dict of SourceEstimate (or VolSourceEstimate)
        The estimated source space induced power estimates in shape
        (n_vertices, n_frequencies, n_samples) if label=None or label=label.
        For lists of one or more labels, the induced power estimate has shape
        (n_labels, n_frequencies, n_samples).
    """  # noqa: E501
    _check_option("method", method, INVERSE_METHODS)

    freqs = np.concatenate(
        [np.arange(band[0], band[1] + df / 2.0, df) for _, band in bands.items()]
    )

    powers, _, vertno = _source_induced_power(
        epochs,
        inverse_operator,
        freqs,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        n_cycles=n_cycles,
        decim=decim,
        use_fft=use_fft,
        pca=pca,
        n_jobs=n_jobs,
        with_plv=False,
        prepared=prepared,
        method_params=method_params,
        use_cps=use_cps,
    )

    Fs = epochs.info["sfreq"]  # sampling in Hz
    stcs = dict()

    subject = _subject_from_inverse(inverse_operator)
    _log_rescale(baseline, baseline_mode)  # for early failure
    for name, band in bands.items():
        idx = [k for k, f in enumerate(freqs) if band[0] <= f <= band[1]]

        # average power in band + mean over epochs
        power = np.mean(powers[:, idx, :], axis=1)

        # Run baseline correction
        power = rescale(
            power,
            epochs.times[::decim],
            baseline,
            baseline_mode,
            copy=False,
            verbose=False,
        )

        tmin = epochs.times[0]
        tstep = float(decim) / Fs
        stc = _make_stc(
            power,
            vertices=vertno,
            tmin=tmin,
            tstep=tstep,
            subject=subject,
            src_type=inverse_operator["src"].kind,
        )
        stcs[name] = stc

        logger.info("[done]")

    return stcs

def _prepare_tfr(data, decim, pick_ori, Ws, K, source_ori):
    """Prepare TFR source localization."""
    n_times = data[:, :, ::decim].shape[2]
    n_freqs = len(Ws)
    n_sources = K.shape[0]
    is_free_ori = False
    if source_ori == FIFF.FIFFV_MNE_FREE_ORI and pick_ori is None:
        is_free_ori = True
        n_sources //= 3

    shape = (n_sources, n_freqs, n_times)
    return shape, is_free_ori

@verbose
def _compute_pow_plv(
    data,
    K,
    sel,
    Ws,
    source_ori,
    use_fft,
    Vh,
    with_power,
    with_plv,
    pick_ori,
    decim,
    noise_norm=None,
    verbose=None,
):
    """Aux function for induced power and PLV."""
    shape, is_free_ori = _prepare_tfr(data, decim, pick_ori, Ws, K, source_ori)
    power = np.zeros(shape, dtype=np.float64)  # power or raw TFR
    # phase lock
    plv = np.zeros(shape, dtype=np.complex128) if with_plv else None

    for epoch in data:
        epoch = epoch[sel]  # keep only selected channels

        if Vh is not None:
            epoch = np.dot(Vh, epoch)  # reducing data rank

        power_e, plv_e = _single_epoch_tfr(
            data=epoch,
            is_free_ori=is_free_ori,
            K=K,
            Ws=Ws,
            use_fft=use_fft,
            decim=decim,
            shape=shape,
            with_plv=with_plv,
            with_power=with_power,
        )

        power += power_e
        if with_plv:
            plv += plv_e

    if noise_norm is not None:
        power *= noise_norm[:, :, np.newaxis] ** 2

    return power, plv

def _single_epoch_tfr(
    data, is_free_ori, K, Ws, use_fft, decim, shape, with_plv, with_power
):
    """Compute single trial TFRs, either ITC, power or raw TFR."""
    tfr_e = np.zeros(shape, dtype=np.float64)  # power or raw TFR
    # phase lock
    plv_e = np.zeros(shape, dtype=np.complex128) if with_plv else None
    n_sources, _, n_times = shape
    for f, w in enumerate(Ws):
        tfr_ = cwt(data, [w], use_fft=use_fft, decim=decim)
        tfr_ = np.asfortranarray(tfr_.reshape(len(data), -1))

        # phase lock and power at freq f
        if with_plv:
            plv_f = np.zeros((n_sources, n_times), dtype=np.complex128)

        tfr_f = np.zeros((n_sources, n_times), dtype=np.float64)

        for k, t in enumerate([np.real(tfr_), np.imag(tfr_)]):
            sol = np.dot(K, t)

            sol_pick_normal = sol
            if is_free_ori:
                sol_pick_normal = sol[2::3]

            if with_plv:
                if k == 0:  # real
                    plv_f += sol_pick_normal
                else:  # imag
                    plv_f += 1j * sol_pick_normal

            if is_free_ori:
                logger.debug("combining the current components...")
                sol = combine_xyz(sol, square=with_power)
            elif with_power:
                sol *= sol
            tfr_f += sol
            del sol

        tfr_e[:, f, :] += tfr_f
        del tfr_f

        if with_plv:
            plv_f /= np.abs(plv_f)
            plv_e[:, f, :] += plv_f
            del plv_f

    return tfr_e, plv_e

def _get_label_power(power, labels, vertno, k_idxs):
    """Average power across vertices in labels."""
    (_, ps1, ps2) = power.shape
    # construct out array with correct shape
    out_power = np.zeros(shape=(len(labels), ps1, ps2))

    # for each label, compile list of vertices we want
    for li in np.arange(len(labels)):
        lab = labels[li]
        hemis = ("lh", "rh")
        all_vnums = [[], []]
        if lab.hemi == "both":
            all_vnums[0] = np.intersect1d(lab.lh.vertices, vertno[0])
            all_vnums[1] = np.intersect1d(lab.rh.vertices, vertno[1])
        else:
            assert lab.hemi == "lh" or lab.hemi == "rh"
            h_id = 0 if lab.hemi == "lh" else 1
            all_vnums[h_id] = np.intersect1d(vertno[h_id], lab.vertices)

        verts = [(hemis[hi], vn) for hi in range(2) for vn in all_vnums[hi]]

        # restrict power to relevant vertices in label
        lab_mask = np.array([False] * len(power))
        for vert in verts:
            lab_mask[k_idxs[vert]] = True  # k_idxs[vert] gives power row index
        lab_power = power[lab_mask]  # only pass through rows we want
        assert lab_power.shape == (len(verts), ps1, ps2)

        # set correct out values for label
        out_power[li, :, :] = np.mean(lab_power, axis=0)

    assert out_power.shape == (len(labels), ps1, ps2)
    return out_power

@verbose
def _source_induced_power(
    epochs,
    inverse_operator,
    freqs,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    n_cycles=5,
    decim=1,
    use_fft=False,
    pca=True,
    pick_ori="normal",
    n_jobs=None,
    with_plv=True,
    zero_mean=False,
    prepared=False,
    method_params=None,
    use_cps=True,
    verbose=None,
):
    """Aux function for source induced power."""
    if label:
        _validate_type(
            label,
            types=(Label, BiHemiLabel, list, tuple, None),
            type_name=("Label or BiHemiLabel", "list of labels", "None"),
        )
        if isinstance(label, (list, tuple)):
            for item in label:
                _validate_type(
                    item,
                    types=(Label, BiHemiLabel),
                    type_name=("Label or BiHemiLabel"),
                )
            if len(label) > 1 and with_plv:
                raise RuntimeError(
                    "Phase-locking value cannot be calculated "
                    "when averaging induced power within "
                    "labels. Please set `with_plv` to False, pass a "
                    "single `label=label`, or set `label=None`."
                )

    epochs_data = epochs.get_data(copy=False)
    K, sel, Vh, vertno, is_free_ori, noise_norm, k_id = _prepare_source_params(
        inst=epochs,
        inverse_operator=inverse_operator,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        pca=pca,
        pick_ori=pick_ori,
        prepared=prepared,
        method_params=method_params,
        use_cps=use_cps,
    )

    inv = inverse_operator
    parallel, my_compute_source_tfrs, n_jobs = parallel_func(
        _compute_pow_plv, n_jobs, max_jobs=len(epochs_data)
    )
    Fs = epochs.info["sfreq"]  # sampling in Hz

    logger.info("Computing source power ...")

    Ws = morlet(Fs, freqs, n_cycles=n_cycles, zero_mean=zero_mean)

    out = parallel(
        my_compute_source_tfrs(
            data=data,
            K=K,
            sel=sel,
            Ws=Ws,
            source_ori=inv["source_ori"],
            use_fft=use_fft,
            Vh=Vh,
            with_plv=with_plv,
            with_power=True,
            pick_ori=pick_ori,
            decim=decim,
            noise_norm=noise_norm,
        )
        for data in np.array_split(epochs_data, n_jobs)
    )
    power = sum(o[0] for o in out)  # power shape: (n_verts, n_freqs, n_samps)
    power /= len(epochs_data)  # average power over epochs

    if isinstance(label, (Label, BiHemiLabel)):
        logger.info(
            f"Outputting power for {len(power)} vertices in label {label.name}."
        )
    elif isinstance(label, list):
        power = _get_label_power(power, label, vertno, k_id)
        logger.info(
            "Averaging induced power across vertices within labels "
            f"for {len(label)} label{_pl(label)}."
        )
    else:
        assert not label

    if with_plv:
        plv = sum(o[1] for o in out)
        plv = np.abs(plv)
        plv /= len(epochs_data)  # average power over epochs
    else:
        plv = None

    return power, plv, vertno

@verbose
def source_induced_power(
    epochs,
    inverse_operator,
    freqs,
    label=None,
    lambda2=1.0 / 9.0,
    method="dSPM",
    nave=1,
    n_cycles=5,
    decim=1,
    use_fft=False,
    pick_ori=None,
    baseline=None,
    baseline_mode="logratio",
    pca=True,
    n_jobs=None,
    *,
    return_plv=True,
    zero_mean=False,
    prepared=False,
    method_params=None,
    use_cps=True,
    verbose=None,
):
    """Compute induced power and phase lock.

    Computation can optionally be restricted in a label.

    Parameters
    ----------
    epochs : instance of Epochs
        The epochs.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    freqs : array
        Array of frequencies of interest.
    label : Label | list of Label
        Restricts the source estimates to a given label or list of labels. If
        labels are provided in a list, power will be averaged over vertices within each
        label.
    lambda2 : float
        The regularization parameter of the minimum norm.
    method : "MNE" | "dSPM" | "sLORETA" | "eLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    n_cycles : float | array of float
        Number of cycles. Fixed number or one per frequency.
    decim : int
        Temporal decimation factor.
    use_fft : bool
        Do convolutions in time or frequency domain with FFT.
    pick_ori : None | "normal"
        If "normal", rather than pooling the orientations by taking the norm,
        only the radial component is kept. This is only implemented
        when working with loose orientations.
    baseline : None (default) or tuple of length 2
        The time interval to apply baseline correction.
        If None do not apply it. If baseline is (a, b)
        the interval is between "a (s)" and "b (s)".
        If a is None the beginning of the data is used
        and if b is None then b is set to the end of the interval.
        If baseline is equal to (None, None) all the time
        interval is used.
    baseline_mode : 'mean' | 'ratio' | 'logratio' | 'percent' | 'zscore' | 'zlogratio'
        Perform baseline correction by

        - subtracting the mean of baseline values ('mean')
        - dividing by the mean of baseline values ('ratio')
        - dividing by the mean of baseline values and taking the log
          ('logratio')
        - subtracting the mean of baseline values followed by dividing by
          the mean of baseline values ('percent')
        - subtracting the mean of baseline values and dividing by the
          standard deviation of baseline values ('zscore')
        - dividing by the mean of baseline values, taking the log, and
          dividing by the standard deviation of log baseline values
          ('zlogratio')

    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    %(n_jobs)s
    return_plv : bool
        If True, return the phase-locking value array. Else, only return power.

        .. versionadded:: 1.6
    zero_mean : bool
        Make sure the wavelets are zero mean.
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.
    %(use_cps_restricted)s

        .. versionadded:: 0.20
    %(verbose)s

    Returns
    -------
    power : array
        The induced power array with shape (n_sources, n_freqs, n_samples) if
        label=None or label=label. For lists of one or more labels, the induced
        power estimate has shape (n_labels, n_frequencies, n_samples).
    plv : array
        The phase-locking value array with shape (n_sources, n_freqs,
        n_samples). Only returned if ``return_plv=True``.
    """  # noqa: E501
    _check_option("method", method, INVERSE_METHODS)
    _check_ori(pick_ori, inverse_operator["source_ori"], inverse_operator["src"])

    power, plv, vertno = _source_induced_power(
        epochs,
        inverse_operator,
        freqs,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        n_cycles=n_cycles,
        decim=decim,
        use_fft=use_fft,
        pick_ori=pick_ori,
        pca=pca,
        n_jobs=n_jobs,
        with_plv=return_plv,
        method_params=method_params,
        zero_mean=zero_mean,
        prepared=prepared,
        use_cps=use_cps,
    )

    # Run baseline correction
    power = rescale(power, epochs.times[::decim], baseline, baseline_mode, copy=False)

    outs = (power, plv) if return_plv else power
    return outs

@verbose
def compute_source_psd(
    raw,
    inverse_operator,
    lambda2=1.0 / 9.0,
    method="dSPM",
    tmin=0.0,
    tmax=None,
    fmin=0.0,
    fmax=200.0,
    n_fft=2048,
    overlap=0.5,
    pick_ori=None,
    label=None,
    nave=1,
    pca=True,
    prepared=False,
    method_params=None,
    inv_split=None,
    bandwidth="hann",
    adaptive=False,
    low_bias=False,
    n_jobs=None,
    return_sensor=False,
    dB=False,
    *,
    verbose=None,
):
    """Compute source power spectral density (PSD).

    Parameters
    ----------
    raw : instance of Raw
        The raw data.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    lambda2 : float
        The regularization parameter.
    method : "MNE" | "dSPM" | "sLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    tmin : float
        The beginning of the time interval of interest (in seconds).
        Use 0. for the beginning of the file.
    tmax : float | None
        The end of the time interval of interest (in seconds). If None
        stop at the end of the file.
    fmin : float
        The lower frequency of interest.
    fmax : float
        The upper frequency of interest.
    n_fft : int
        Window size for the FFT. Should be a power of 2.
    overlap : float
        The overlap fraction between windows. Should be between 0 and 1.
        0 means no overlap.
    pick_ori : None | "normal"
        If "normal", rather than pooling the orientations by taking the norm,
        only the radial component is kept. This is only implemented
        when working with loose orientations.
    label : Label
        Restricts the source estimates to a given label.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.

        .. versionadded:: 0.16
    inv_split : int or None
        Split inverse operator into inv_split parts in order to save memory.

        .. versionadded:: 0.17
    bandwidth : float | str
        The bandwidth of the multi taper windowing function in Hz.
        Can also be a string (e.g., 'hann') to use a single window.

        For backward compatibility, the default is 'hann'.

        .. versionadded:: 0.17
    adaptive : bool
        Use adaptive weights to combine the tapered spectra into PSD
        (slow, use n_jobs >> 1 to speed up computation).

        .. versionadded:: 0.17
    low_bias : bool
        Only use tapers with more than 90%% spectral concentration within
        bandwidth.

        .. versionadded:: 0.17
    %(n_jobs)s
        It is only used if adaptive=True.

        .. versionadded:: 0.17
    return_sensor : bool
        If True, return the sensor PSDs as an EvokedArray.

        .. versionadded:: 0.17
    dB : bool
        If True (default False), return output it decibels.

        .. versionadded:: 0.17
    %(verbose)s

    Returns
    -------
    stc_psd : instance of SourceEstimate | VolSourceEstimate
        The PSD of each of the sources.
    sensor_psd : instance of EvokedArray
        The PSD of each sensor. Only returned if ``return_sensor`` is True.

    See Also
    --------
    compute_source_psd_epochs

    Notes
    -----
    Each window is multiplied by a window before processing, so
    using a non-zero overlap is recommended.

    This function is different from :func:`compute_source_psd_epochs` in that:

    1. ``bandwidth='hann'`` by default, skipping multitaper estimation
    2. For convenience it wraps
       :func:`mne.make_fixed_length_events` and :class:`mne.Epochs`.

    Otherwise the two should produce identical results.
    """
    tmin = 0.0 if tmin is None else float(tmin)
    overlap = float(overlap)
    if not 0 <= overlap < 1:
        raise ValueError(
            "Overlap must be at least 0 and less than 1, got %s" % (overlap,)
        )
    n_fft = int(n_fft)
    duration = ((1.0 - overlap) * n_fft) / raw.info["sfreq"]
    events = make_fixed_length_events(raw, 1, tmin, tmax, duration)
    epochs = Epochs(raw, events, 1, 0, (n_fft - 1) / raw.info["sfreq"], baseline=None)
    out = compute_source_psd_epochs(
        epochs,
        inverse_operator,
        lambda2,
        method,
        fmin,
        fmax,
        pick_ori,
        label,
        nave,
        pca,
        inv_split,
        bandwidth,
        adaptive,
        low_bias,
        True,
        n_jobs,
        prepared,
        method_params,
        return_sensor=True,
    )
    source_data = 0.0
    sensor_data = 0.0
    count = 0
    for stc, evoked in out:
        source_data += stc.data
        sensor_data += evoked.data
        count += 1
    assert count > 0  # should be guaranteed by make_fixed_length_events
    sensor_data /= count
    source_data /= count
    if dB:
        np.log10(sensor_data, out=sensor_data)
        sensor_data *= 10.0
        np.log10(source_data, out=source_data)
        source_data *= 10.0
    evoked.data = sensor_data
    evoked.nave = count
    stc.data = source_data
    out = stc
    if return_sensor:
        out = (out, evoked)
    return out

def _compute_source_psd_epochs(
    epochs,
    inverse_operator,
    lambda2=1.0 / 9.0,
    method="dSPM",
    fmin=0.0,
    fmax=200.0,
    pick_ori=None,
    label=None,
    nave=1,
    pca=True,
    inv_split=None,
    bandwidth=4.0,
    adaptive=False,
    low_bias=True,
    n_jobs=None,
    prepared=False,
    method_params=None,
    return_sensor=False,
    use_cps=True,
):
    """Generate compute_source_psd_epochs."""
    logger.info("Considering frequencies %g ... %g Hz" % (fmin, fmax))

    if label:
        # TODO: add multi-label support
        # since `_prepare_source_params` can handle a list of labels now,
        # multi-label support should be within reach for psd calc as well
        _validate_type(
            label,
            types=(Label, BiHemiLabel, None),
            type_name=("Label or BiHemiLabel", "None"),
        )

    K, sel, Vh, vertno, is_free_ori, noise_norm, _ = _prepare_source_params(
        inst=epochs,
        inverse_operator=inverse_operator,
        label=label,
        lambda2=lambda2,
        method=method,
        nave=nave,
        pca=pca,
        pick_ori=pick_ori,
        prepared=prepared,
        method_params=method_params,
        use_cps=use_cps,
    )
    # Simplify code with a tiny (rel. to other computations) penalty for eye
    # mult
    Vh = np.eye(K.shape[1]) if Vh is None else Vh

    # split the inverse operator
    if inv_split is not None:
        K_split = np.array_split(K, inv_split)
    else:
        K_split = [K]

    # compute DPSS windows
    n_times = len(epochs.times)
    sfreq = epochs.info["sfreq"]

    dpss, eigvals, adaptive = _compute_mt_params(
        n_times, sfreq, bandwidth, low_bias, adaptive, verbose=False
    )

    n_tapers = len(dpss)
    try:
        n_epochs = len(epochs)
    except RuntimeError:
        n_epochs = len(epochs.events)
        extra = "on at most %d epochs" % (n_epochs,)
    else:
        extra = "on %d epochs" % (n_epochs,)
    if isinstance(bandwidth, str):
        bandwidth = "%s windowing" % (bandwidth,)
    else:
        bandwidth = "%d tapers with bandwidth %0.1f Hz" % (n_tapers, bandwidth)
    logger.info("Using %s %s" % (bandwidth, extra))

    if adaptive:
        parallel, my_psd_from_mt_adaptive, n_jobs = parallel_func(
            _psd_from_mt_adaptive, n_jobs
        )
    else:
        weights = np.sqrt(eigvals)[np.newaxis, :, np.newaxis]

    subject = _subject_from_inverse(inverse_operator)
    iter_epochs = ProgressBar(epochs, max_value=n_epochs)
    evoked_info = pick_info(epochs.info, sel, verbose=False)
    for k, e in enumerate(iter_epochs):
        data = np.dot(Vh, e[sel])  # reducing data rank

        # compute tapered spectra in sensor space
        x_mt, freqs = _mt_spectra(data, dpss, sfreq)

        if k == 0:
            freq_mask = (freqs >= fmin) & (freqs <= fmax)
            fstep = np.mean(np.diff(freqs))
            with evoked_info._unlock():
                evoked_info["sfreq"] = 1.0 / fstep
        freqs = freqs[freq_mask]

        # sensor space PSD
        x_mt_sensor = np.empty(
            (len(sel), x_mt.shape[1], x_mt.shape[2]), dtype=x_mt.dtype
        )
        for i in range(n_tapers):
            x_mt_sensor[:, i, :] = np.dot(Vh.T, x_mt[:, i, :])
        if adaptive:
            out = parallel(
                my_psd_from_mt_adaptive(x, eigvals, freq_mask)
                for x in np.array_split(x_mt_sensor, min(n_jobs, len(x_mt_sensor)))
            )
            sensor_psd = np.concatenate(out)
        else:
            x_mt_sensor = x_mt_sensor[:, :, freq_mask]
            sensor_psd = _psd_from_mt(x_mt_sensor, weights)

        # allocate space for output
        psd = np.empty((K.shape[0], np.sum(freq_mask)))

        # Optionally, we split the inverse operator into parts to save memory.
        # Without splitting the tapered spectra in source space have size
        # (n_vertices x n_tapers x n_times / 2)
        pos = 0
        for K_part in K_split:
            # allocate space for tapered spectra in source space
            x_mt_src = np.empty(
                (K_part.shape[0], x_mt.shape[1], x_mt.shape[2]), dtype=x_mt.dtype
            )

            # apply inverse to each taper (faster than equiv einsum)
            for i in range(n_tapers):
                x_mt_src[:, i, :] = np.dot(K_part, x_mt[:, i, :])

            # compute the psd
            if adaptive:
                out = parallel(
                    my_psd_from_mt_adaptive(x, eigvals, freq_mask)
                    for x in np.array_split(x_mt_src, min(n_jobs, len(x_mt_src)))
                )
                this_psd = np.concatenate(out)
            else:
                x_mt_src = x_mt_src[:, :, freq_mask]
                this_psd = _psd_from_mt(x_mt_src, weights)

            psd[pos : pos + K_part.shape[0], :] = this_psd
            pos += K_part.shape[0]

        # combine orientations
        if is_free_ori and pick_ori is None:
            psd = combine_xyz(psd, square=False)

        if noise_norm is not None:
            psd *= noise_norm**2

        out = _make_stc(
            psd,
            tmin=freqs[0],
            tstep=fstep,
            vertices=vertno,
            subject=subject,
            src_type=inverse_operator["src"].kind,
        )

        if return_sensor:
            comment = "Epoch %d PSD" % (k,)
            out = (
                out,
                EvokedArray(sensor_psd, evoked_info.copy(), freqs[0], comment, nave),
            )

        # we return a generator object for "stream processing"
        yield out

    iter_epochs.update(n_epochs)  # in case some were skipped

@verbose
def compute_source_psd_epochs(
    epochs,
    inverse_operator,
    lambda2=1.0 / 9.0,
    method="dSPM",
    fmin=0.0,
    fmax=200.0,
    pick_ori=None,
    label=None,
    nave=1,
    pca=True,
    inv_split=None,
    bandwidth=4.0,
    adaptive=False,
    low_bias=True,
    return_generator=False,
    n_jobs=None,
    prepared=False,
    method_params=None,
    return_sensor=False,
    use_cps=True,
    verbose=None,
):
    """Compute source power spectral density (PSD) from Epochs.

    This uses the multi-taper method to compute the PSD for each epoch.

    Parameters
    ----------
    epochs : instance of Epochs
        The raw data.
    inverse_operator : instance of InverseOperator
        The inverse operator.
    lambda2 : float
        The regularization parameter.
    method : "MNE" | "dSPM" | "sLORETA" | "eLORETA"
        Use minimum norm, dSPM (default), sLORETA, or eLORETA.
    fmin : float
        The lower frequency of interest.
    fmax : float
        The upper frequency of interest.
    pick_ori : None | "normal"
        If "normal", rather than pooling the orientations by taking the norm,
        only the radial component is kept. This is only implemented
        when working with loose orientations.
    label : Label
        Restricts the source estimates to a given label.
    nave : int
        The number of averages used to scale the noise covariance matrix.
    pca : bool
        If True, the true dimension of data is estimated before running
        the time-frequency transforms. It reduces the computation times
        e.g. with a dataset that was maxfiltered (true dim is 64).
    inv_split : int or None
        Split inverse operator into inv_split parts in order to save memory.
    bandwidth : float | str
        The bandwidth of the multi taper windowing function in Hz.
        Can also be a string (e.g., 'hann') to use a single window.
    adaptive : bool
        Use adaptive weights to combine the tapered spectra into PSD
        (slow, use n_jobs >> 1 to speed up computation).
    low_bias : bool
        Only use tapers with more than 90%% spectral concentration within
        bandwidth.
    return_generator : bool
        Return a generator object instead of a list. This allows iterating
        over the stcs without having to keep them all in memory.
    %(n_jobs)s
        It is only used if adaptive=True.
    prepared : bool
        If True, do not call :func:`prepare_inverse_operator`.
    method_params : dict | None
        Additional options for eLORETA. See Notes of :func:`apply_inverse`.

        .. versionadded:: 0.16
    return_sensor : bool
        If True, also return the sensor PSD for each epoch as an EvokedArray.

        .. versionadded:: 0.17
    %(use_cps_restricted)s

        .. versionadded:: 0.20
    %(verbose)s

    Returns
    -------
    out : list (or generator object)
        A list (or generator) for the source space PSD (and optionally the
        sensor PSD) for each epoch.

    See Also
    --------
    compute_source_psd
    """
    # use an auxiliary function so we can either return a generator or a list
    stcs_gen = _compute_source_psd_epochs(
        epochs,
        inverse_operator,
        lambda2=lambda2,
        method=method,
        fmin=fmin,
        fmax=fmax,
        pick_ori=pick_ori,
        label=label,
        nave=nave,
        pca=pca,
        inv_split=inv_split,
        bandwidth=bandwidth,
        adaptive=adaptive,
        low_bias=low_bias,
        n_jobs=n_jobs,
        prepared=prepared,
        method_params=method_params,
        return_sensor=return_sensor,
        use_cps=use_cps,
    )

    if return_generator:
        # return generator object
        return stcs_gen
    else:
        # return a list
        stcs = list()
        for stc in stcs_gen:
            stcs.append(stc)

        return stcs


# coding=utf-8

import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
import uuid
import json
from django.shortcuts import render, render_to_response
from django.http import HttpResponse
from cms.models import Picture, Classification
from django.views.decorators.csrf import csrf_exempt
from PIL import Image

basePath = os.path.dirname(os.path.dirname(__file__))

@csrf_exempt
def getPicturePosition(req):
	"""
	获取所属栏目
	:param req:
	:return:
	"""
	classification = Classification.objects.all().order_by("classificationIdLevel")    #根据栏目级别排序
	List = []
	for obj in classification:
		classificationList = {}
		classificationList["classificationId"] = obj.classificationId
		classificationList["classificationName"] = obj.classificationName
		classificationList["classificationIdLevel"] = obj.classificationIdLevel
		List.append(classificationList)
	return HttpResponse(json.dumps(List))

@csrf_exempt
def savePicture(req):
	"""
	添加图片
	:param req:
	:return:
	"""
	# picture = Picture.objects.filter(pictureId = req.POST["pictureId"])
	picturePosition = req.POST["picturePosition"]          #所属栏目
	pictureName = req.POST["pictureName"]                  #图片名称
	isShow = req.POST["isShow"]                            #是否启用
	if isShow == "true":
		isShow = 1
	else:
		isShow = 0
	#保存图片
	reqfile = req.FILES.get("pictureUrl", False)
	if reqfile != False:
		pictureUrl = "cms/static/img/classPic/"+pictureName
		picturePath = os.path.join(basePath, pictureUrl)
		img = Image.open(reqfile)
		img.thumbnail((700, 700), Image.ANTIALIAS)         #对图片进行等比缩放
		img.save(picturePath, "png")                      #保存图片
	else:
		pictureUrl = ""
	#修改图片
	#添加图片
	pictureId = uuid.uuid1()
	picture = Picture(pictureId=pictureId, pictureName=pictureName, picturePath="/"+pictureUrl, isShow=isShow, classificationId_id=picturePosition)
	picture.save()
	return HttpResponse(1)


import numpy as np
from scipy.signal import savgol_filter
if __name__ == '__main__':
    import sys
    from os import path
    sys.path.append( path.dirname(path.dirname( path.abspath(__file__) ) ))

from lib.OpenChromFile import FileOpenClass
from lib.Smoother import Moving_average, data_Bunching
from lib import AirPLS
from lib.peakDetectionAlgorithm import adjustPeaksBoundary, peakSearchAlgorithm
from lib.thresholdCalc import threshold

from lib.ChromAlgorithm import SmoothData, Detrand, data_Bunching
from lib.Peak import Peak
from lib.PeakSymmetry import calcPeakSymmetry

'''
class Peak:
    # ------------ Property ------------
    # Peak 邊界 (front x, front y, back x, back y)
    boundary = (0,0,0,0)
    boundary_index = (-1,-1) # ( front index, back index)
    # Peak 峰值位置 (x,y)
    Apex = (0,0)
    Apex_index = -1
    # Peak 基本參數
    height,tr, width ,area = 0,0,0,0
    # peak 的非對稱資訊
    w05,w01,w005,tf,As,N, sigma= 0,0,0,1,1,0,0
    # implied Peak 資訊 ( 三次微分所找的peak )
    co_peak = []
    #-----------------------------------
    def __init__(self):
        pass
'''
class Chromatograph:
    # ------------ Property ------------
    time=[]
    signal=[]
    baseline=[]
    # 微分圖譜, 分別為一次微分, 二次微分, 三次微分
    dydx=[]
    dydx2=[]
    dydx3=[]
    # 儲存 Peaks 的容器 , 為 list(Class Peak)
    peak_list=[]
    # -----------------------------------

    def __init__(self, time=[], signal=[]):
        self.time = time
        self.signal = signal

    def DataBunch(self, bunching_point=3):
        self.time, self.signal = data_Bunching( self.time, self.signal, bunching_point )

    def CalcBaseline(self, method="AirPLS", *args, **kwargs):
        lambda_ = kwargs.get('lambda_') if 'lambda_' in kwargs else 10000
        self.baseline = Detrand( method, self.signal, lambda_= lambda_ )

    def Smooth( self, method='Savitzky_Golay_Smooth', window_size=13 , *args, **kwargs ):
        order = kwargs.get('order') if 'order' in kwargs else 3
        mode = kwargs.get('mode') if 'mode' in kwargs else 'nearest'
        sigma = kwargs.get('sigma') if 'sigma' in kwargs else 7

        self.time, self.signal = SmoothData(
            method,
            self.time,
            self.signal,
            window_size,
            order=order,
            mode= mode,
            sigma=sigma
        )

    def SmoothDerivative( self, window_size=21, alpha=1.5):
        # ------ 把 window size 處理為奇數 ------
        window_size = window_size+1 if window_size%2 == 0 else window_size
        # ------ ------ ------ ------ ----- -----
        dx = self.time[1] - self.time[0]

        self.dydx = savgol_filter(
            self.signal, window_length=window_size, polyorder=2, deriv=1, delta=dx
          )  # 一次微分
        self.dydx2 = savgol_filter(
            self.signal, window_length=int(window_size*alpha), polyorder=2, deriv=2, delta=dx
            ) # 二次微分
        self.dydx3 = savgol_filter(
            self.signal, window_length=int(window_size*alpha*alpha), polyorder=3, deriv=3, delta=dx
            ) # 三次微分

    def PeakDetection(self, k = 10, tail_factor = 0.3, MIN_PEAK_INTERVAL=0.5 ):
        dydx_threshold = threshold(self.dydx.copy(), k)
        dydx2_threshold = threshold(self.dydx2.copy(), k)
        #///////////////////////////////////////////////////////////////////////
        peak_index_list = peakSearchAlgorithm(
            [self.time,self.signal],
            [self.dydx,self.dydx2,self.dydx3],
            dydx_threshold[1],
            dydx_threshold[0] * tail_factor,
            dydx2_threshold[0],
            offset=10,
            MAX_TIME_WIDTH=0.5
            )

        # peak_index_list : list  , 2D-array
        # array( [ start index, highest index, end index, apex index ] )
        # index 為 time array 或 signal array的編號
        # start index : peak 起始點的 index
        # highest index : peak 最高點的 index
        # end index : peak 終點的 index
        # apex index : 峰值位置的 index, 為 list, 為利用三次微分圖譜所找到的 peak
        #   為單一peak時, len(apex index)=1且應為 apex index[0] == highest index
        #   若有高度共析狀況發生, 圖譜與一次微分無發區分該共析peaks,
        #   此時 len(apex index) > 1, 其 list內容則是共析peak的apex位置
        #///////////////////////////////////////////////////////////////////////

        boundary_table = adjustPeaksBoundary(
            self.time, self.signal, peak_index_list, self.baseline,
            MIN_PEAK_INTERVAL
            )
        self.__build_peak_list( peak_index_list, boundary_table )

        #-----------------------

        temp = []
        for p in self.peak_list:
            temp.append(
                calcPeakSymmetry(self.time, self.signal, p)
                )
        self.peak_list = temp.copy()

        #------------------------

    def __build_peak_list(self, peak_index_list, boundary_table):
        self.peak_list = []
        for i in range( len(peak_index_list) ):
            peak = Peak()
            peak.boundary_index = (
                peak_index_list[i][0], peak_index_list[i][2]
                )
            peak.boundary= ( boundary_table[i] )
            peak.Apex = (
                self.time[ peak_index_list[i][1]], self.signal[ peak_index_list[i][1] ]
                )
            peak.Apex_index = peak_index_list[i][1]

            peak.height = peak.Apex[1]
            peak.tr = peak.Apex[0]
            peak.width = peak.boundary[2] - peak.boundary[0]
            # ----- 計算積分 -----
            area = np.trapz(
                self.signal[ peak.boundary_index[0]: peak.boundary_index[1] ],
                self.time[ peak.boundary_index[0]: peak.boundary_index[1] ]
                )

            # ----- 計算基線 -----
            # 因為計算出來的 baseline可能會大於原始訊號, 這會造成積分值會比預期低
            # 因此多一到工序  min( signal, baseline ), 保證積分時的基線必 <= signal
            base = []
            for i in range( peak.boundary_index[0], peak.boundary_index[1], 1 ):
                base.append( min( self.signal[i], self.baseline[i] ) )

            base_area = np.trapz(
                base ,
                self.time[ peak.boundary_index[0]: peak.boundary_index[1] ]
                )
            peak.area = area - base_area

            # ------ ------ ------ ------
            peak.co_peak = peak_index_list[ 3 ]

            # ----- 寫入資料 -----
            self.peak_list.append( peak )
            # ------ ------ ------ ------

    def peakHeightFilter( self, SN_rate=3.0 ):
        # 利用 peak height 過濾"已找尋到"的 peaks
        new_peak_list = []
        for peak in self.peak_list:
            if (peak.height / self.baseline[ peak.Apex_index ] ) >= SN_rate :
                new_peak_list.append( peak )
        self.peak_list = new_peak_list


# -*- coding: utf-8 -*-
import base64

SCRIPT_SETS = {
    "search_host_files": "config_query/job_scripts/search_host_files",
    "backup_host_files": "config_query/job_scripts/backup_host_files",
}

def get_script_base64(script_name, params=""):
    """
    获取对应脚本的base64编码
    """
    with open(SCRIPT_SETS[script_name], "r", encoding="utf-8") as f:
        script_content = f.read()

    return base64.b64encode(script_content.encode("utf8")).decode("utf-8"), base64.b64encode(
        params.encode("utf8")
    ).decode("utf-8")


import os
from urllib.request import Request, urlopen

FILE_NAME_DELIMITER = '_'
FILE_NAME_SUFFIX = '_TBD.url'
FILTER_FILE_NAME_SUFFIX = 'Filter_TBD.url'
REWRITE_FILE_NAME_SUFFIX = 'Rewrite_TBD.url'
RULE_SET_FILE_NAME_SUFFIX = 'RuleSet_TBD.url'
RESULTS_DIR = 'results'
RESULT_FILE_NAME_SUFFIX = 'Integrated.list'
PROVIDER_FILE_NAME_SUFFIX = 'Provider.yaml'

FILTER_RESULT = {}
REWRITE_RESULT = {}

FILTER_SR = []
REWRITE_SR = []
EXCLUDE_LIST = []

def read_list(url_list_2b, file_name_2b='', src_mark_flag=True):
    global FILTER_RESULT, REWRITE_RESULT
    result = {}
    hostname = {}
    for list_url in url_list_2b:
        list_url = list_url.strip()
        if list_url.startswith('http'):
            try:
                headers = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
                                  ' AppleWebKit/537.36 (KHTML, like Gecko) '
                                  'Chrome/86.0.4240.111 Safari/537.36'}
                req = Request(list_url, headers=headers)
                response_list = urlopen(req).read()
                response_list = str(response_list, encoding='UTF-8').split('\n')
                if src_mark_flag:
                    result[f'# {list_url}'] = None
                for line_list in response_list:
                    line_list = line_list.encode('ascii', errors='ignore').decode('ascii').strip()
                    if line_list.__len__() > 0 and (not line_list.startswith('#')):
                        ex_flag = False
                        qx_sr_flag = False
                        for ex_list_entry in EXCLUDE_LIST:
                            if ex_list_entry in line_list:
                                ex_flag = True
                                break
                        if ex_flag:
                            continue
                        if file_name_2b.endswith(FILTER_FILE_NAME_SUFFIX):
                            if (line_list.startswith('DOMAIN')
                                    or line_list.startswith('DOMAIN-KEYWORD')
                                    or line_list.startswith('USER-AGENT')
                                    or line_list.startswith('IP-CIDR')
                                    or line_list.startswith('HOST')
                                    or line_list.startswith('HOST-SUFFIX')
                                    or line_list.startswith('HOST-KEYWORD')):
                                if file_name_2b.startswith('Anti_AD'):
                                    line_list = rule_processor(line_list, action='REJECT')
                                for special_rule in FILTER_SR:
                                    if (not (special_rule in list_url)) and (special_rule in line_list):
                                        qx_sr_flag = True
                                        break
                                if line_list in FILTER_RESULT:
                                    continue
                                else:
                                    FILTER_RESULT[line_list] = None
                            else:
                                continue
                        elif file_name_2b.endswith(REWRITE_FILE_NAME_SUFFIX):
                            if line_list.startswith('hostname = '):
                                line_hostname = line_list[10:line_list.__len__()]
                                line_hostname_list = line_hostname.split(',')
                                for entry_hostname in line_hostname_list:
                                    useless_hostname = False
                                    for special_rule in REWRITE_SR:
                                        if (not (special_rule in list_url)) and (special_rule in entry_hostname):
                                            useless_hostname = True
                                            break
                                    if not useless_hostname:
                                        hostname[entry_hostname.strip()] = None
                                continue
                            for special_rule in REWRITE_SR:
                                if (not (special_rule in list_url)) and (special_rule in line_list):
                                    qx_sr_flag = True
                                    break
                            if line_list in REWRITE_RESULT:
                                continue
                            else:
                                REWRITE_RESULT[line_list] = None
                        elif file_name_2b.endswith(RULE_SET_FILE_NAME_SUFFIX):
                            line_list = rule_processor(line_list)
                        if qx_sr_flag:
                            continue
                        else:
                            result[line_list] = None
            except Exception as get_fail:
                print(f'Error: Fail to get rule from {list_url} \n {get_fail} ')
    if len(hostname) > 0:
        result_hostname = 'hostname = '
        for entry_hostname in hostname.keys():
            result_hostname = f'{result_hostname}{entry_hostname},'
        result_hostname = result_hostname.rstrip(',')
        result[result_hostname] = None
    return result

def read_qx_sr_list(qx_sr_dir_path, qx_sr_dir):
    qx_sr_filter_url_list = []
    qx_sr_rewrite_url_list = []
    qx_sr_results_path = path_processor(qx_sr_dir_path)
    if not os.path.exists(f'{RESULTS_DIR}{qx_sr_results_path}/qx'):
        os.makedirs(f'{RESULTS_DIR}{qx_sr_results_path}/qx')
    qx_sr_2b_dir_input = os.walk(f'{qx_sr_dir_path}/{qx_sr_dir}')
    for qx_sr_path, qx_sr_dir_list, qx_sr_file_list in qx_sr_2b_dir_input:
        for qx_sr_file_name in qx_sr_file_list:
            os.system(f'echo {qx_sr_file_name}')
            if qx_sr_file_name.endswith(FILE_NAME_SUFFIX):
                qx_sr_url_list = []
                with open(f'{qx_sr_dir_path}/{qx_sr_dir}/{qx_sr_file_name}', mode='r', encoding='UTF-8') as qx_sr_file_content:
                    for qx_sr_url_list_entry in qx_sr_file_content.readlines():
                        qx_sr_url_list.append(qx_sr_url_list_entry)
                if qx_sr_file_name.endswith(REWRITE_FILE_NAME_SUFFIX):
                    REWRITE_SR.append(qx_sr_file_name[0:qx_sr_file_name.index(FILE_NAME_DELIMITER)])
                    qx_sr_rewrite_url_list = qx_sr_rewrite_url_list + qx_sr_url_list
                elif qx_sr_file_name.endswith(FILTER_FILE_NAME_SUFFIX):
                    FILTER_SR.append(qx_sr_file_name[0:qx_sr_file_name.index(FILE_NAME_DELIMITER)])
                    qx_sr_filter_url_list = qx_sr_filter_url_list + qx_sr_url_list
    if qx_sr_rewrite_url_list.__len__() > 0:
        with open(f'{RESULTS_DIR}{qx_sr_results_path}/qx/QX_SR_Rewrite_{RESULT_FILE_NAME_SUFFIX}', mode='w', encoding='UTF-8') as qx_sr_results:
            qx_sr_dic_results = read_list(qx_sr_rewrite_url_list, REWRITE_FILE_NAME_SUFFIX)
            for qx_sr_key in qx_sr_dic_results.keys():
                qx_sr_results.write(f'{qx_sr_key}\n')
            qx_sr_results.flush()
    if qx_sr_filter_url_list.__len__() > 0:
        with open(f'{RESULTS_DIR}{qx_sr_results_path}/qx/QX_SR_Filter_{RESULT_FILE_NAME_SUFFIX}', mode='w', encoding='UTF-8') as qx_sr_results:
            qx_sr_dic_results = read_list(qx_sr_filter_url_list, FILTER_FILE_NAME_SUFFIX)
            for qx_sr_key in qx_sr_dic_results.keys():
                qx_sr_results.write(f'{qx_sr_key}\n')
            qx_sr_results.flush()
    return

def read_qx_nr_list(qx_nr_dir_path, qx_nr_dir):
    qx_nr_results_path = path_processor(qx_nr_dir_path)
    if not os.path.exists(f'{RESULTS_DIR}{qx_nr_results_path}/qx'):
        os.makedirs(f'{RESULTS_DIR}{qx_nr_results_path}/qx')
    qx_nr_2b_dir_input = os.walk(f'{qx_nr_dir_path}/{qx_nr_dir}')
    for qx_nr_path, qx_nr_dir_list, qx_nr_file_list in qx_nr_2b_dir_input:
        for qx_nr_file_name in qx_nr_file_list:
            os.system(f'echo {qx_nr_file_name}')
            if qx_nr_file_name.endswith(FILE_NAME_SUFFIX):
                qx_nr_url_list = []
                with open(f'{qx_nr_path}/{qx_nr_file_name}', mode='r', encoding='UTF-8') as qx_nr_file_content:
                    for qx_nr_url_list_entry in qx_nr_file_content.readlines():
                        qx_nr_url_list.append(qx_nr_url_list_entry)
                qx_nr_file_name_new = f'{qx_nr_file_name[0:qx_nr_file_name.index(FILE_NAME_SUFFIX)]}_{RESULT_FILE_NAME_SUFFIX}'
                with open(f'{RESULTS_DIR}{qx_nr_results_path}/qx/{qx_nr_file_name_new}', mode='w', encoding='UTF-8') as qx_nr_results:
                    qx_nr_dic_results = read_list(qx_nr_url_list, file_name_2b=qx_nr_file_name)
                    for qx_nr_key in qx_nr_dic_results.keys():
                        qx_nr_results.write(f'{qx_nr_key}\n')
                    qx_nr_results.flush()
    return

def read_rs_list(rs_dir_path, rs_dir):
    rs_results_path = path_processor(rs_dir_path)
    if not os.path.exists(f'{RESULTS_DIR}{rs_results_path}/rules'):
        os.makedirs(f'{RESULTS_DIR}{rs_results_path}/rules')
    if not os.path.exists(f'{RESULTS_DIR}{rs_results_path}/clash'):
        os.makedirs(f'{RESULTS_DIR}{rs_results_path}/clash')
    rs_2b_dir_input = os.walk(f'{rs_dir_path}/{rs_dir}')
    for rs_path, rs_dir_list, rs_file_list in rs_2b_dir_input:
        for rs_file_name in rs_file_list:
            os.system(f'echo {rs_file_name}')
            if rs_file_name.endswith(RULE_SET_FILE_NAME_SUFFIX):
                rs_url_list = []
                with open(f'{rs_dir_path}/{rs_dir}/{rs_file_name}', mode='r', encoding='UTF-8') as rs_file_content:
                    for rs_url_list_entry in rs_file_content.readlines():
                        rs_url_list.append(rs_url_list_entry)
                    rs_file_name_fragment = rs_file_name[0:rs_file_name.index(FILE_NAME_SUFFIX)]
                    with open(f'{RESULTS_DIR}{rs_results_path}/rules/{rs_file_name_fragment}_{RESULT_FILE_NAME_SUFFIX}', mode='w', encoding='UTF-8') as rs_results:
                        with open(f'{RESULTS_DIR}{rs_results_path}/clash/{rs_file_name_fragment}_{PROVIDER_FILE_NAME_SUFFIX}', mode='w', encoding='UTF-8') as scd_results:
                            scd_results.write(f'payload:\n  # > {rs_file_name}\n')
                            rs_dic_results = read_list(rs_url_list, file_name_2b=rs_file_name, src_mark_flag=False)
                            for rs_key in rs_dic_results.keys():
                                rs_results.write(f'{rs_key}\n')
                                if (rs_key.startswith('DOMAIN')
                                        or rs_key.startswith('DOMAIN-KEYWORD')
                                        or rs_key.startswith('DOMAIN-SUFFIX')
                                        or rs_key.startswith('IP-CIDR')
                                        or rs_key.startswith('SRC-IP-CIDR')
                                        or rs_key.startswith('GEOIP')
                                        or rs_key.startswith('PROCESS-NAME')
                                        or rs_key.startswith('DST-PORT')
                                        or rs_key.startswith('SRC-PORT')
                                        or rs_key.startswith('MATCH')):
                                    scd_results.write(f'  - {rs_key}\n')
                            scd_results.flush()
                        rs_results.flush()
    return

special_dir_switch = {
    'QX_NR': read_qx_nr_list,
    'QX_SR': read_qx_sr_list,
    'RULE_SET': read_rs_list
}

def rule_processor(rule_line, action=''):
    rule_line_fragment = rule_line.split(',')
    if rule_line_fragment.__len__() > 2 and (not rule_line_fragment[2] == 'no-resolve'):
        if action.__len__() > 0:
            rule_line_fragment[2] = action
        else:
            rule_line_fragment.pop(2)
        rule_line_new = ''
        for fragment in rule_line_fragment:
            rule_line_new = f'{rule_line_new}{fragment.strip()},'
        return rule_line_new[0:rule_line_new.__len__() - 1]
    else:
        return rule_line

def path_processor(src_path):
    if '/' in src_path:
        return src_path[src_path.index('/'):src_path.__len__()]
    if '\\' in src_path:
        return src_path[src_path.index('\\'):src_path.__len__()].replace('\\', '/')
    return ''

if __name__ == '__main__':
    list_2b_dir_input = os.walk('url_TBD')
    if not os.path.exists(RESULTS_DIR):
        os.makedirs(RESULTS_DIR)
    for path, dir_list, file_list in list_2b_dir_input:
        if os.path.exists(f'{path}/EXCLUDE'):
            with open(f'{path}/EXCLUDE', mode='r', encoding='UTF-8') as ex_file_content:
                for ex_url_list_entry in ex_file_content.readlines():
                    EXCLUDE_LIST.append(ex_url_list_entry)
        dir_list.sort(reverse=True)
        for special_dir in dir_list:
            # noinspection PyBroadException
            try:
                special_dir_switch[special_dir](path, special_dir)
            except Exception as dir_error:
                continue


import json
import requests
from easygui import choicebox, enterbox

from helper import project_api_id, access_token, base_url

title = "JamaScript"
display = ""
artifact_result = ""
main_components_result = ""
component_ids = []
test_management_ids = []
target_item_id = 0
user_choice = 0
main_selection = 0
main2_selection = 0
total_items_imported = 0
original_count = 0
choices = ["Architecture", "Business Request", "Component", "Data Dictionary",
           "Data Requirement", "Defect", "Epic", "Functional Requirement",
           "Meeting Note", "Non-Functional Requirement", "Persona", "Requirement",
           "Style Guide", "Test Case", "Text", "Use Case", "User Storie", "Wireframe"]
headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"}

def add_components_to_root_level(root_cmp_names1):
    global original_count, main_components_result
    original_count = 9 * len(root_cmp_names1)
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    for name in root_cmp_names1:
        data = {"project": project_api_id, "itemType": 30,
                "childItemType": 30, "fields": {"name": name}}

        response = requests.request("POST", url, headers=headers, data=json.dumps(data))
        global component_ids
        component_ids.append(response.json()["meta"]["id"])
    main_components_result += ("[" + str(len(component_ids)) + "/" + str(len(root_cmp_names1)) + "] Components "
                                                                                "created\n")
    global total_items_imported
    total_items_imported += len(component_ids)

def add_sets_to_components():
    count = []
    child_item_types = ["25", "87", "87", "87", "116"]
    set_names = ["Use Cases", "Functional Requirements", "Non-Functional Requirements", "Technical Requirements",
                 "Wireframes"]
    set_keys = ["UC", "FR", "NFR", "TR", "WF"]
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    i = 0
    while 5 > i > -1:
        for item in component_ids:
            data = {"project": project_api_id, "itemType": 31,
                    "childItemType": child_item_types[i], "location":
                        {
                            "parent":
                                {
                                    "item": item
                                }
                        }, "fields": {"setKey": set_keys[i], "name": set_names[i]}
                    }
            response = requests.request("POST", url, headers=headers, data=json.dumps(data))
            count.append(response.json()["meta"]["id"])
        i += 1
    global main_components_result
    main_components_result += ("[" + str(5 * len(component_ids)) + "/" + str(len(count)) + "] Sets of Use Cases, "
                                                                                "Functional "
                                                                                "Requirements, "
                                                                                "Non-Functional "
                                                                                "Requirements, Technical "
                                                                                "Requirements, "
                                                                                "and Wireframes created\n")
    global total_items_imported
    total_items_imported += (5 * len(component_ids))

def add_test_management():
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    for item in component_ids:
        data = {"project": project_api_id, "itemType": 30,
                "childItemType": 30, "location": {"parent": {"item": item}}, "fields": {"name": "Test Management"}}
        response = requests.request("POST", url, headers=headers, data=json.dumps(data))
        global test_management_ids
        test_management_ids.append(response.json()["meta"]["id"])
    global main_components_result
    main_components_result += ("[" + str(len(component_ids)) + "/" + str(len(test_management_ids)) + "] Test "
                                                                                "Management Sub "
                                                                                "Components "
                                                                                "created\n")
    global total_items_imported
    total_items_imported += len(component_ids)

def add_test_cases_and_defects():
    count = []
    child_item_types = ["26", "27"]
    set_names = ["Test Cases", "Defects"]
    set_keys = ["TC", "DEF"]
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    i = 0
    while 2 > i > -1:
        for item in test_management_ids:
            data = {"project": project_api_id,
                    "itemType": 31,
                    "childItemType": child_item_types[i],
                    "location": {
                        "parent": {
                            "item": item}
                    }, "fields": {"setKey": set_keys[i], "name": set_names[i]}
                    }
            response = requests.request("POST", url, headers=headers, data=json.dumps(data))
            count.append(response.json()["meta"]["id"])
        i += 1
    global main_components_result, total_items_imported
    main_components_result += ("[" + str(len(count)) + "/" + str(2 * len(component_ids)) + "] Sets of Test Cases and "
                                                                                "Defects created\n")
    total_items_imported += (2 * len(component_ids))
    if original_count == total_items_imported:
        main_components_result += ("\nSuccess, imported [" + str(total_items_imported) + "] items!")
    else:
        main_components_result += ("Failed, only [" + str(total_items_imported) + "] items were imported.")

def get_user_choices():
    global user_choice
    user_choice = choicebox("What artifact type would you like to add?", choices=choices)
    if user_choice == "Go back":
        main()
    else:
        msg = "Please enter the target item's ID"
        document_key = enterbox(msg, title)
        url = base_url + "/rest/latest/abstractitems?project=" + str(project_api_id) + "&documentKey=" + document_key
        response = requests.request("GET", url, headers=headers)
        global target_item_id
        target_item_id = response.json()["data"][0]['id']

def add_uc(uc_data1):
    count = []
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    for name, pre, main_flow, post, alt, blueprint_id in zip(uc_data1["Name"], uc_data1["PreCondition"],
                                                             uc_data1["MainFlow"],
                                                             uc_data1["PostCondition"], uc_data1["AlternateFlows"],
                                                             uc_data1["Blueprint_ID"]):
        data = {"project": project_api_id, "itemType": 25,
                "childItemType": 25, "location":
                    {
                        "parent":
                            {
                                "item": target_item_id
                            }
                    }, "fields": {"name": name, "precondition": pre,
                                  "mainflow": main_flow, "postcondition": post, "alternateflows": alt,
                                  "blueprint_id": blueprint_id}
                }
        response = requests.request("POST", url, headers=headers, data=json.dumps(data))
        count.append(response.json()["meta"]["id"])
    global artifact_result
    if len(uc_data1["Name"]) == len(count):
        artifact_result = "Success, imported " + str(len(count)) + " Use Cases!"
    else:
        artifact_result = "Failed, only " + str(len(count)) + " Use Cases imported"

def add_cmp(cmp_name1):
    count = []
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    for name in cmp_name1:
        data = {"project": project_api_id, "itemType": 30,
                "childItemType": 30, "location":
                    {
                        "parent":
                            {
                                "item": target_item_id
                            }
                    }, "fields": {"name": name}
                }
        response = requests.request("POST", url, headers=headers, data=json.dumps(data))
        count.append(response.json()["meta"]["id"])
    global artifact_result
    if len(cmp_name1) == len(count):
        artifact_result = "Success, imported [" + str(len(count)) + "] Components!"
    else:
        artifact_result = "Failed, only [" + str(len(count)) + "] Components imported."

def add_rq(rq_data1):
    count = []
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    for name, description, blueprint_id in zip(rq_data1["Name"], rq_data1["Description"], rq_data1["Blueprint_ID"]):
        data = {"project": project_api_id, "itemType": 87,
                "childItemType": 87, "location":
                    {
                        "parent":
                            {
                                "item": target_item_id
                            }
                    }, "fields": {"name": name, "description": description, "blueprint_id": blueprint_id}
                }
        response = requests.request("POST", url, headers=headers, data=json.dumps(data))
        count.append(response.json()["meta"]["id"])
    global artifact_result
    if len(rq_data1["Name"]) == len(count):
        artifact_result = "Success, imported [" + str(len(count)) + "] Requirements!"
    else:
        artifact_result = "Failed, only [" + str(len(count)) + "] Requirements were imported."

def add_wf(wf_data1):
    count = []
    url = base_url + "/rest/latest/items?setGlobalIdManually=false"
    for name, blueprint_id in zip(wf_data1["Name"], wf_data1["Blueprint_ID"]):
        data = {"project": project_api_id, "itemType": 116,
                "childItemType": 116, "location":
                    {
                        "parent":
                            {
                                "item": target_item_id
                            }
                    }, "fields": {"name": name, "blueprint_id": blueprint_id}
                }
        response = requests.request("POST", url, headers=headers, data=json.dumps(data))
        count.append(response.json()["meta"]["id"])
    global artifact_result
    if len(wf_data1["Name"]) == len(count):
        artifact_result = "Success, imported [" + str(len(count)) + "] Wireframes!"
    else:
        artifact_result = "Failed, only [" + str(len(count)) + "] Wireframes imported."

def add_requirements():
    global display
    if user_choice == choices[0]:
        display = "Architecture in development."
        main2()
    elif user_choice == choices[1]:
        display = "Business Request in development."
        main2()
    elif user_choice == choices[2]:
        from file_handler import components_file
        add_cmp(components_file())
        display = artifact_result
        main2()
    elif user_choice == choices[3]:
        display = "Data Dictionary in development."
        main2()
    elif user_choice == choices[4]:
        display = "Data Requirement in development."
        main2()
    elif user_choice == choices[5]:
        display = "Defect in development."
        main2()
    elif user_choice == choices[6]:
        display = "Epic in development."
        main2()
    elif user_choice == choices[7]:
        display = "Functional Requirement in development."
        main2()
    elif user_choice == choices[8]:
        display = "Meeting Note in development."
        main2()
    elif user_choice == choices[9]:
        display = "Non-Functional Requirement in development."
        main2()
    elif user_choice == choices[10]:
        display = "Persona in development."
        main2()
    elif user_choice == choices[11]:
        from file_handler import requirements_file
        add_rq(requirements_file())
        display = artifact_result
        main2()
    elif user_choice == choices[12]:
        display = "Style Guide in development."
        main2()
    elif user_choice == choices[13]:
        display = "Test Case in development."
        main2()
    elif user_choice == choices[14]:
        display = "Text in development."
        main2()
    elif user_choice == choices[15]:
        from file_handler import use_cases_file
        add_uc(use_cases_file())
        display = artifact_result
        main2()
    elif user_choice == choices[16]:
        display = "User Story in development."
        main2()
    elif user_choice == choices[17]:
        from file_handler import wireframes_file
        add_wf(wireframes_file())
        display = artifact_result
        main2()
    else:
        print("Error")
        get_user_choices()

def main2():
    choices = ["Add artifacts", "Change project"]
    global main2_selection
    main2_selection = choicebox(display + "\n\n What else would you like to do?", choices=choices, title=title)
    if main2_selection == "Add artifacts":
        get_user_choices()
        add_requirements()
    elif main2_selection == "Change project":
        from helper import get_project_name
        get_project_name()
        main()
    else:
        exit()

def main():
    choices = ["Add main components with their sets", "Add artifacts", "Change project"]
    global main_selection, display
    main_selection = choicebox("What would you like to do?", choices=choices, title=title)
    if main_selection == "Add main components with their sets":
        from file_handler import components_file
        add_components_to_root_level(components_file())
        add_sets_to_components()
        add_test_management()
        add_test_cases_and_defects()
        display = main_components_result
        main2()
    elif main_selection == "Add artifacts":
        get_user_choices()
        add_requirements()
        display = artifact_result
    elif main_selection == "Change project":
        from helper import get_project_name
        get_project_name()
        main()
    else:
        exit()

if __name__ == '__main__':
    main()


from transformers import AutoModel, AutoTokenizer
import torch
import torch.nn.functional as F
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import numpy as np
import pandas as pd
import os,gc,re,warnings
import sys

from cuml.svm import SVR
import cuml

#Fetching data from folds
dftr = pd.read_csv('C:/Users/lionh/OneDrive/Desktop/roberta-train/data/train_folds.csv')
dfts = pd.read_csv('C:/Users/lionh/OneDrive/Desktop/roberta-train/data/test.csv')

#for ease I have defined the models here, but youb need to take it from config in actual process
tokenizer = None
MAX_LEN = 640
BATCH_SIZE = 4
model = {
    'MODEL_LM_debertabase' : 'microsoft/deberta-base',
    'MODEL_LM_debertalargev3' : 'microsoft/deberta-v3-large',
    'MODEL_LM_debertalarge' : 'microsoft/deberta-large',
    'MODEL_LM_debertalargemnli' : 'microsoft/deberta-large-mnli',
    'MODEL_LM_debertaxlarge' : 'microsoft/deberta-xlarge'
}

#Code for mean pooling
def mean_pooling(model_output, attention_mask):
    token_embeddings = model_output.last_hidden_state.detach().cpu()
    input_mask_expanded = (
        attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
    )
    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(
        input_mask_expanded.sum(1), min=1e-9
    )

class EmbedDataset(torch.utils.data.Dataset):
    def __init__(self,df):
        self.df = df.reset_index(drop=True)
    def __len__(self):
        return len(self.df)
    def __getitem__(self,idx):
        text = self.df.loc[idx,"full_text"]
        tokens = tokenizer(
                text,
                None,
                add_special_tokens=True,
                padding='max_length',
                truncation=True,
                max_length=MAX_LEN,return_tensors="pt")
        tokens = {k:v.squeeze(0) for k,v in tokens.items()}
        return tokens

ds_tr = EmbedDataset(dftr)
embed_dataloader_tr = torch.utils.data.DataLoader(ds_tr,\
                        batch_size=BATCH_SIZE,\
                        shuffle=False)
ds_te = EmbedDataset(dfts)
embed_dataloader_te = torch.utils.data.DataLoader(ds_te,\
                        batch_size=BATCH_SIZE,\
                        shuffle=False)

#Extracting Embeddings from various deberta models

def get_embeddings(MODEL_NM='', MAX=640, BATCH_SIZE=4, verbose=True):
    global tokenizer, MAX_LEN
    DEVICE="cuda"
    model = AutoModel.from_pretrained( MODEL_NM )
    tokenizer = AutoTokenizer.from_pretrained( MODEL_NM )
    MAX_LEN = MAX

    model = model.to(DEVICE)
    model.eval()
    all_train_text_feats = []
    for batch in tqdm(embed_dataloader_tr,total=len(embed_dataloader_tr)):
        input_ids = batch["input_ids"].to(DEVICE)
        attention_mask = batch["attention_mask"].to(DEVICE)
        with torch.no_grad():
            model_output = model(input_ids=input_ids,attention_mask=attention_mask)
        sentence_embeddings = mean_pooling(model_output, attention_mask.detach().cpu())
        # Normalize the embeddings
        sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
        sentence_embeddings =  sentence_embeddings.squeeze(0).detach().cpu().numpy()
        all_train_text_feats.extend(sentence_embeddings)
    all_train_text_feats = np.array(all_train_text_feats)
    if verbose:
        print('Train embeddings shape',all_train_text_feats.shape)

    te_text_feats = []
    for batch in tqdm(embed_dataloader_te,total=len(embed_dataloader_te)):
        input_ids = batch["input_ids"].to(DEVICE)
        attention_mask = batch["attention_mask"].to(DEVICE)
        with torch.no_grad():
            model_output = model(input_ids=input_ids,attention_mask=attention_mask)
        sentence_embeddings = mean_pooling(model_output, attention_mask.detach().cpu())
        # Normalize the embeddings
        sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
        sentence_embeddings =  sentence_embeddings.squeeze(0).detach().cpu().numpy()
        te_text_feats.extend(sentence_embeddings)
    te_text_feats = np.array(te_text_feats)
    if verbose:
        print('Test embeddings shape',te_text_feats.shape)

    return all_train_text_feats, te_text_feats

#taking deberta variants one by one
MODEL_NM = model['MODEL_LM_debertabase']
all_train_text_feats, te_text_feats = get_embeddings(MODEL_NM)
print('Got debertabase embeddings')

MODEL_NM = model['MODEL_LM_debertalargev3']
all_train_text_feats2, te_text_feats2 = get_embeddings(MODEL_NM)

MODEL_NM = model['MODEL_LM_debertalarge']
all_train_text_feats2, te_text_feats2 = get_embeddings(MODEL_NM)

MODEL_NM = model['MODEL_LM_debertalargemnli']
all_train_text_feats2, te_text_feats2 = get_embeddings(MODEL_NM)

MODEL_NM = model['MODEL_LM_debertaxlarge']
all_train_text_feats2, te_text_feats2 = get_embeddings(MODEL_NM)

#combinig all the embeddings
all_train_text_feats = np.concatenate([all_train_text_feats,all_train_text_feats2,
                                       all_train_text_feats3,all_train_text_feats4,
                                       all_train_text_feats5],axis=1)

te_text_feats = np.concatenate([te_text_feats,te_text_feats2,
                                te_text_feats3,te_text_feats4,
                                te_text_feats5],axis=1)

#deleting all the variables to free up memory
del all_train_text_feats2, te_text_feats2
del all_train_text_feats3, te_text_feats3
del all_train_text_feats4, te_text_feats4
del all_train_text_feats5, te_text_feats5
gc.collect()

print('concatenated embeddings have shape', all_train_text_feats.shape)

#Now defining a Rapid SVR model to predict the target variable
preds = []
scores = []
def comp_score(y_true,y_pred):
    rmse_scores = []
    for i in range(len(target_cols)):
        rmse_scores.append(np.sqrt(mean_squared_error(y_true[:,i],y_pred[:,i])))
    return np.mean(rmse_scores)

#for fold in tqdm(range(FOLDS),total=FOLDS):
for fold in range(FOLDS):
    print('#'*25)
    print('### Fold',fold+1)
    print('#'*25)

    dftr_ = dftr[dftr["FOLD"]!=fold]
    dfev_ = dftr[dftr["FOLD"]==fold]

    tr_text_feats = all_train_text_feats[list(dftr_.index),:]
    ev_text_feats = all_train_text_feats[list(dfev_.index),:]

    ev_preds = np.zeros((len(ev_text_feats),6))
    test_preds = np.zeros((len(te_text_feats),6))
    for i,t in enumerate(target_cols):
        print(t,', ',end='')
        clf = SVR(C=1)
        clf.fit(tr_text_feats, dftr_[t].values)
        ev_preds[:,i] = clf.predict(ev_text_feats)
        test_preds[:,i] = clf.predict(te_text_feats)
    print()
    score = comp_score(dfev_[target_cols].values,ev_preds)
    scores.append(score)
    print("Fold : {} RSME score: {}".format(fold,score))
    preds.append(test_preds)

print('#'*25)
print('Overall CV RSME =',np.mean(scores))

#preds: output of the model
#  array([[2.94900918, 2.80713797, 3.1332984 , 2.95393324, 2.66409802,
#          2.67784142],
#         [2.72251678, 2.46881819, 2.70102072, 2.32998419, 2.05252576,
#          2.68055749],
#         [3.63700128, 3.44540453, 3.56421232, 3.65185833, 3.40497899,
#          3.35660815]])]
# np.array(preds)
# [[2.91515398 2.82003951 3.16167831 2.97052073 2.67681551 2.68956351]
#   [2.70253325 2.45898795 2.71900558 2.3118124  2.01470613 2.63774276]
#   [3.6399312  3.4656961  3.58689547 3.64418197 3.41387224 3.35341716]]

#taking average of the predictions
sub = dfte.copy()
sub.loc[:,target_cols] = np.average(np.array(preds),axis=0)
sub_columns = pd.read_csv("C:/Users/lionh/OneDrive/Desktop/roberta-train/data/sample_submission.csv").columns
sub = sub[sub_columns]