import os

import fixtures
from oslo.config import cfg

from nova.openstack.common import uuidutils
from nova import test
from nova.tests import fake_libvirt_utils
from nova.tests import fake_utils
from nova.virt.libvirt import imagebackend

CONF = cfg.CONF

class _ImageTestCase(object):
    INSTANCES_PATH = '/instances_path'

    def mock_create_image(self, image):
        def create_image(fn, base, size, *args, **kwargs):
            fn(target=base, *args, **kwargs)
        image.create_image = create_image

    def setUp(self):
        super(_ImageTestCase, self).setUp()
        self.flags(disable_process_locking=True,
                   instances_path=self.INSTANCES_PATH)
        self.INSTANCE = {'name': 'instance',
                         'uuid': uuidutils.generate_uuid()}
        self.NAME = 'fake.vm'
        self.TEMPLATE = 'template'

        self.OLD_STYLE_INSTANCE_PATH = \
            fake_libvirt_utils.get_instance_path(self.INSTANCE, forceold=True)
        self.PATH = os.path.join(
            fake_libvirt_utils.get_instance_path(self.INSTANCE), self.NAME)

        # TODO(mikal): rename template_dir to base_dir and template_path
        # to cached_image_path. This will be less confusing.
        self.TEMPLATE_DIR = os.path.join(CONF.instances_path, '_base')
        self.TEMPLATE_PATH = os.path.join(self.TEMPLATE_DIR, 'template')

        self.useFixture(fixtures.MonkeyPatch(
            'nova.virt.libvirt.imagebackend.libvirt_utils',
            fake_libvirt_utils))

    def test_cache(self):
        self.mox.StubOutWithMock(os.path, 'exists')
        if self.OLD_STYLE_INSTANCE_PATH:
            os.path.exists(self.OLD_STYLE_INSTANCE_PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_DIR).AndReturn(False)
        os.path.exists(self.PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_PATH).AndReturn(False)
        fn = self.mox.CreateMockAnything()
        fn(target=self.TEMPLATE_PATH)
        self.mox.StubOutWithMock(imagebackend.fileutils, 'ensure_tree')
        imagebackend.fileutils.ensure_tree(self.TEMPLATE_DIR)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        self.mock_create_image(image)
        image.cache(fn, self.TEMPLATE)

        self.mox.VerifyAll()

    def test_cache_image_exists(self):
        self.mox.StubOutWithMock(os.path, 'exists')
        if self.OLD_STYLE_INSTANCE_PATH:
            os.path.exists(self.OLD_STYLE_INSTANCE_PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_DIR).AndReturn(True)
        os.path.exists(self.PATH).AndReturn(True)
        os.path.exists(self.TEMPLATE_PATH).AndReturn(True)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.cache(None, self.TEMPLATE)

        self.mox.VerifyAll()

    def test_cache_base_dir_exists(self):
        self.mox.StubOutWithMock(os.path, 'exists')
        if self.OLD_STYLE_INSTANCE_PATH:
            os.path.exists(self.OLD_STYLE_INSTANCE_PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_DIR).AndReturn(True)
        os.path.exists(self.PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_PATH).AndReturn(False)
        fn = self.mox.CreateMockAnything()
        fn(target=self.TEMPLATE_PATH)
        self.mox.StubOutWithMock(imagebackend.fileutils, 'ensure_tree')
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        self.mock_create_image(image)
        image.cache(fn, self.TEMPLATE)

        self.mox.VerifyAll()

    def test_cache_template_exists(self):
        self.mox.StubOutWithMock(os.path, 'exists')
        if self.OLD_STYLE_INSTANCE_PATH:
            os.path.exists(self.OLD_STYLE_INSTANCE_PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_DIR).AndReturn(True)
        os.path.exists(self.PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_PATH).AndReturn(True)
        fn = self.mox.CreateMockAnything()
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        self.mock_create_image(image)
        image.cache(fn, self.TEMPLATE)

        self.mox.VerifyAll()

    def test_prealloc_image(self):
        CONF.set_override('preallocate_images', 'space')

        fake_utils.fake_execute_clear_log()
        fake_utils.stub_out_utils_execute(self.stubs)
        image = self.image_class(self.INSTANCE, self.NAME)

        def fake_fetch(target, *args, **kwargs):
            return

        self.stubs.Set(os.path, 'exists', lambda _: True)

        # Call twice to verify testing fallocate is only called once.
        image.cache(fake_fetch, self.TEMPLATE_PATH, self.SIZE)
        image.cache(fake_fetch, self.TEMPLATE_PATH, self.SIZE)

        self.assertEqual(fake_utils.fake_execute_get_log(),
            ['fallocate -n -l 1 %s.fallocate_test' % self.PATH,
             'fallocate -n -l %s %s' % (self.SIZE, self.PATH),
             'fallocate -n -l %s %s' % (self.SIZE, self.PATH)])

class RawTestCase(_ImageTestCase, test.TestCase):

    SIZE = 1024

    def setUp(self):
        self.image_class = imagebackend.Raw
        super(RawTestCase, self).setUp()
        self.stubs.Set(imagebackend.Raw, 'correct_format', lambda _: None)

    def prepare_mocks(self):
        fn = self.mox.CreateMockAnything()
        self.mox.StubOutWithMock(imagebackend.lockutils.synchronized,
                                 '__call__')
        self.mox.StubOutWithMock(imagebackend.libvirt_utils, 'copy_image')
        self.mox.StubOutWithMock(imagebackend.disk, 'extend')
        return fn

    def test_create_image(self):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH, image_id=None)
        imagebackend.libvirt_utils.copy_image(self.TEMPLATE_PATH, self.PATH)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, None, image_id=None)

        self.mox.VerifyAll()

    def test_create_image_generated(self):
        fn = self.prepare_mocks()
        fn(target=self.PATH)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, None)

        self.mox.VerifyAll()

    def test_create_image_extend(self):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH, image_id=None)
        imagebackend.libvirt_utils.copy_image(self.TEMPLATE_PATH, self.PATH)
        imagebackend.disk.extend(self.PATH, self.SIZE)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, self.SIZE, image_id=None)

        self.mox.VerifyAll()

    def test_correct_format(self):
        info = self.mox.CreateMockAnything()
        self.stubs.UnsetAll()

        self.mox.StubOutWithMock(os.path, 'exists')
        self.mox.StubOutWithMock(imagebackend.images, 'qemu_img_info')

        os.path.exists(self.PATH).AndReturn(True)
        info = self.mox.CreateMockAnything()
        info.file_format = 'foo'
        imagebackend.images.qemu_img_info(self.PATH).AndReturn(info)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME, path=self.PATH)
        self.assertEqual(image.driver_format, 'foo')

        self.mox.VerifyAll()

class Qcow2TestCase(_ImageTestCase, test.TestCase):
    SIZE = 1024 * 1024 * 1024

    def setUp(self):
        self.image_class = imagebackend.Qcow2
        super(Qcow2TestCase, self).setUp()
        self.QCOW2_BASE = (self.TEMPLATE_PATH +
                           '_%d' % (self.SIZE / (1024 * 1024 * 1024)))

    def prepare_mocks(self):
        fn = self.mox.CreateMockAnything()
        self.mox.StubOutWithMock(imagebackend.lockutils.synchronized,
                                 '__call__')
        self.mox.StubOutWithMock(imagebackend.libvirt_utils,
                                 'create_cow_image')
        self.mox.StubOutWithMock(imagebackend.libvirt_utils, 'copy_image')
        self.mox.StubOutWithMock(imagebackend.disk, 'extend')
        return fn

    def test_create_image(self):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH)
        imagebackend.libvirt_utils.create_cow_image(self.TEMPLATE_PATH,
                                                    self.PATH)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, None)

        self.mox.VerifyAll()

    def test_create_image_with_size(self):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH)
        self.mox.StubOutWithMock(os.path, 'exists')
        if self.OLD_STYLE_INSTANCE_PATH:
            os.path.exists(self.OLD_STYLE_INSTANCE_PATH).AndReturn(False)
        os.path.exists(self.TEMPLATE_PATH).AndReturn(False)
        os.path.exists(self.PATH).AndReturn(False)
        imagebackend.libvirt_utils.create_cow_image(self.TEMPLATE_PATH,
                                                    self.PATH)
        imagebackend.disk.extend(self.PATH, self.SIZE)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, self.SIZE)

        self.mox.VerifyAll()

class LvmTestCase(_ImageTestCase, test.TestCase):
    VG = 'FakeVG'
    TEMPLATE_SIZE = 512
    SIZE = 1024

    def setUp(self):
        self.image_class = imagebackend.Lvm
        super(LvmTestCase, self).setUp()
        self.flags(libvirt_images_volume_group=self.VG)
        self.LV = '%s_%s' % (self.INSTANCE['name'], self.NAME)
        self.OLD_STYLE_INSTANCE_PATH = None
        self.PATH = os.path.join('/dev', self.VG, self.LV)

        self.disk = imagebackend.disk
        self.utils = imagebackend.utils
        self.libvirt_utils = imagebackend.libvirt_utils

    def prepare_mocks(self):
        fn = self.mox.CreateMockAnything()
        self.mox.StubOutWithMock(self.disk, 'resize2fs')
        self.mox.StubOutWithMock(self.libvirt_utils, 'create_lvm_image')
        self.mox.StubOutWithMock(self.disk, 'get_disk_size')
        self.mox.StubOutWithMock(self.utils, 'execute')
        return fn

    def _create_image(self, sparse):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH)
        self.libvirt_utils.create_lvm_image(self.VG,
                                            self.LV,
                                            self.TEMPLATE_SIZE,
                                            sparse=sparse)
        self.disk.get_disk_size(self.TEMPLATE_PATH
                                         ).AndReturn(self.TEMPLATE_SIZE)
        cmd = ('qemu-img', 'convert', '-O', 'raw', self.TEMPLATE_PATH,
               self.PATH)
        self.utils.execute(*cmd, run_as_root=True)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, None)

        self.mox.VerifyAll()

    def _create_image_generated(self, sparse):
        fn = self.prepare_mocks()
        self.libvirt_utils.create_lvm_image(self.VG, self.LV,
                                            self.SIZE, sparse=sparse)
        fn(target=self.PATH, ephemeral_size=None)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH,
                self.SIZE, ephemeral_size=None)

        self.mox.VerifyAll()

    def _create_image_resize(self, sparse):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH)
        self.libvirt_utils.create_lvm_image(self.VG, self.LV,
                                            self.SIZE, sparse=sparse)
        self.disk.get_disk_size(self.TEMPLATE_PATH
                                         ).AndReturn(self.TEMPLATE_SIZE)
        cmd = ('qemu-img', 'convert', '-O', 'raw', self.TEMPLATE_PATH,
               self.PATH)
        self.utils.execute(*cmd, run_as_root=True)
        self.disk.resize2fs(self.PATH, run_as_root=True)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)
        image.create_image(fn, self.TEMPLATE_PATH, self.SIZE)

        self.mox.VerifyAll()

    def test_create_image(self):
        self._create_image(False)

    def test_create_image_sparsed(self):
        self.flags(libvirt_sparse_logical_volumes=True)
        self._create_image(True)

    def test_create_image_generated(self):
        self._create_image_generated(False)

    def test_create_image_generated_sparsed(self):
        self.flags(libvirt_sparse_logical_volumes=True)
        self._create_image_generated(True)

    def test_create_image_resize(self):
        self._create_image_resize(False)

    def test_create_image_resize_sparsed(self):
        self.flags(libvirt_sparse_logical_volumes=True)
        self._create_image_resize(True)

    def test_create_image_negative(self):
        fn = self.prepare_mocks()
        fn(target=self.TEMPLATE_PATH)
        self.libvirt_utils.create_lvm_image(self.VG,
                                            self.LV,
                                            self.SIZE,
                                            sparse=False
                                            ).AndRaise(RuntimeError())
        self.disk.get_disk_size(self.TEMPLATE_PATH
                                         ).AndReturn(self.TEMPLATE_SIZE)
        self.mox.StubOutWithMock(self.libvirt_utils, 'remove_logical_volumes')
        self.libvirt_utils.remove_logical_volumes(self.PATH)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)

        self.assertRaises(RuntimeError, image.create_image, fn,
                          self.TEMPLATE_PATH, self.SIZE)
        self.mox.VerifyAll()

    def test_create_image_generated_negative(self):
        fn = self.prepare_mocks()
        fn(target=self.PATH,
           ephemeral_size=None).AndRaise(RuntimeError())
        self.libvirt_utils.create_lvm_image(self.VG,
                                            self.LV,
                                            self.SIZE,
                                            sparse=False)
        self.mox.StubOutWithMock(self.libvirt_utils, 'remove_logical_volumes')
        self.libvirt_utils.remove_logical_volumes(self.PATH)
        self.mox.ReplayAll()

        image = self.image_class(self.INSTANCE, self.NAME)

        self.assertRaises(RuntimeError, image.create_image, fn,
                          self.TEMPLATE_PATH, self.SIZE,
                          ephemeral_size=None)
        self.mox.VerifyAll()

    def test_prealloc_image(self):
        CONF.set_override('preallocate_images', 'space')

        fake_utils.fake_execute_clear_log()
        fake_utils.stub_out_utils_execute(self.stubs)
        image = self.image_class(self.INSTANCE, self.NAME)

        def fake_fetch(target, *args, **kwargs):
            return

        self.stubs.Set(os.path, 'exists', lambda _: True)

        image.cache(fake_fetch, self.TEMPLATE_PATH, self.SIZE)

        self.assertEqual(fake_utils.fake_execute_get_log(), [])

class BackendTestCase(test.TestCase):
    INSTANCE = {'name': 'fake-instance',
                'uuid': uuidutils.generate_uuid()}
    NAME = 'fake-name.suffix'

    def get_image(self, use_cow, image_type):
        return imagebackend.Backend(use_cow).image(self.INSTANCE,
                                                   self.NAME,
                                                   image_type)

    def _test_image(self, image_type, image_not_cow, image_cow):
        image1 = self.get_image(False, image_type)
        image2 = self.get_image(True, image_type)

        def assertIsInstance(instance, class_object):
            failure = ('Expected %s,' +
                       ' but got %s.') % (class_object.__name__,
                                          instance.__class__.__name__)
            self.assertTrue(isinstance(instance, class_object), failure)

        assertIsInstance(image1, image_not_cow)
        assertIsInstance(image2, image_cow)

    def test_image_raw(self):
        self._test_image('raw', imagebackend.Raw, imagebackend.Raw)

    def test_image_qcow2(self):
        self._test_image('qcow2', imagebackend.Qcow2, imagebackend.Qcow2)

    def test_image_lvm(self):
        self.flags(libvirt_images_volume_group='FakeVG')
        self._test_image('lvm', imagebackend.Lvm, imagebackend.Lvm)

    def test_image_default(self):
        self._test_image('default', imagebackend.Raw, imagebackend.Qcow2)


from QualtricsData import QualtricsData
from pprint import pprint
import pandas as pd

test_datafile_path = '../data/raw/example_data_file.csv'
test_value_codebook_path = '../data/codebook/example_value_codebook.csv'
test_var_codebook_path = '../data/codebook/example_var_codebook.csv'

qd = QualtricsData(test_datafile_path)

def rename_recode():
    qd.rename_vars(test_var_codebook_path)
    qd.recode_values(test_value_codebook_path)

def test_rename_vars():
    qd.rename_vars(test_var_codebook_path)

def test__set_value_codebook():
    qd.rename_vars(test_var_codebook_path)
    qd._set_value_codebook(test_value_codebook_path)

def test_recode_values():
    qd.rename_vars(test_var_codebook_path)
    qd.recode_values(test_value_codebook_path)

def test_filter_values():
    rename_recode()
    qd.filter_rows({'difficulty_2': 3})

if __name__ == '__main__':
    # test_recode_values()
    print(qd.df_proc.shape)
    test_filter_values()
    print(qd.df_proc['difficulty_2'])
    print(qd.df_proc.shape)


def save_model(model_path, word_dict):
    f = open(model_path, 'w')
    f.write(str(word_dict))
    f.close()

def load_model(model_path):
    f = open(model_path,'r')
    a = f.read()
    word_dict = eval(a)
    f.close()
    return word_dict

def init(word_dict):
    word_trans = {}
    for word in word_dict:
        word_trans[word] = {}
    return word_trans

if __name__ == '__main__':
    word_dict = load_model('./model/set_list.txt')
    word_trans = init(word_dict)
    count_dict = {}

    for line in open('../data/train.txt', encoding='utf-8'):
        words_list = []
        if line:
            line = line.strip()
            wl = line.split(' ')
            for w in wl:
                word,tag = w.split('/')
                words_list.append(word)
                if word not in count_dict:
                    count_dict[word] = 1
                else:
                    count_dict[word] += 1

            for i in range(len(words_list) - 1):
                if words_list[i+1] not in word_trans[words_list[i]]:
                    word_trans[words_list[i]][words_list[i + 1]] = 1
                else:
                    word_trans[words_list[i]][words_list[i + 1]] += 1
    save_model('./model/word_trans.txt', word_trans)
    save_model('./model/count_dict.txt', word_trans)
    for key in word_trans.keys():
        for key1 in word_trans[key].keys():
            word_trans[key][key1] = word_trans[key][key1]/count_dict[key]

    save_model('./model/prob_word_trans.txt',word_trans)


# *-* coding:utf8 *-*
# 打开文件
old_file = open("文档操作/文档_dome", encoding="utf-8")
new_file = open("文档操作/文档_dome(复件)", "w", encoding="utf-8")

# 读、写文件
while True:
    read_file = old_file.readline()
    # 判断是否读取到内容
    if read_file == "":
        break
    # 将读取到的内容写入new_file
    new_file.write(read_file)

# 关闭文件
old_file.close()
new_file.close()

日常练习/文档操作/复制大文件.py


import luigi
import os
import nestedtext as nt
import shutil
import pandas as pd
import hashlib
import subprocess
import gzip
import tarfile
from misc import *
from getglobals import PARAMSET_GLOBALS

class INSTINCT_pipeline:

    #run() defined in specific pipeline

    def __init__(self,params,n,pipe_args,upstream,namespace,compdef,n_ref,dag):

        #print(n)

        self.params=params
        self.n=n
        self.n_ref=n_ref
        self.namespace = namespace

        if pipe_args!={}:

            pipe_args = pipelink_unpack(pipe_args) #contains a hidden 'if' to check for pipe_link

            if 'params_drop' in pipe_args:
                #print("******did params drop!*******")
                #import code
                #code.interact(local=dict(globals(), **locals()))
                #print(self.pipeID)
                #print(pipe_args['params_drop'])
                #print("******did params drop!*******")
                #print(pipe_args['params_drop'])
                self.params = param_smoosh(self.params,pipe_args['params_drop'])
                #print(self.params)

            #test if .pipe is referenced in pipeline#
            pipe_args,pipe_val,compdef,pipeID =pipetest_pargs_pval_compdef(pipe_args,namespace)

            #print(pipe_args)
            self.pipe_args=pipe_args[self.__class__.__name__] #drop to name of pipeline in pipe_args

        #print(self.pipe_args)
        self.upstream = upstream

        self.pipeID = pipeID

        #pass copies, so the originals aren't modified up the pipe
        self.params_copy = self.params.copy()
        self.pipe_args_copy = self.pipe_args.copy()

        #compdef will be specified as an attribute for defined pipelines (the not .pipe syntax version)

        self.compdef=compdef
        self.dag = dag

        #import code
        #code.interact(local=dict(globals(), **locals()))

    def run_component(self,component=None,final_process=False,upstream=[None]):

        #print(component)
        #print(self.n)
        #print(self.n_ref)

        if final_process==True:
            #print(self.namespace[self.pipe_args['process']])
                    #if cls.__name__ == "DLmodel_Test":
                        #import code
                        #code.interact(local=dict(globals(), **locals()))
            #
            component_name = self.pipe_args['process']
            n_use = pipeline_n_extract(self.n,self.n_ref,component_name)

            #print(n_use)
            #print(self.n_ref)
            #print(self.namespace[component_name])

            #I changed the params passing from self.params_copy to self.params... keep an eye on other ramifications of this change.
            process = self.namespace[component_name].invoke(self.params_copy,n=n_use,n_ref=self.n_ref,pipe_args=self.pipe_args_copy,compdef=None,\
                                                                            upstream=upstream,pipeID=self.pipeID,namespace = self.namespace,dag=self.dag)
            #print(self.pipe_args_copy)
            #print(self.params_copy["Job"]["DLmodel*"])
            #if self.pipe_args_copy['process'] == "DLmodel_Test":
            #    import code
            #    code.interact(local=dict(globals(), **locals()))
            return process
        elif component in self.pipe_args:

            #here, test if pipe or process
            #if component =="GetDETx":

            #assign temporary pipe args so any modifications will not be retained
            pipe_args = self.pipe_args_copy.copy()

            #test- similar to above.
            params = self.params_copy.copy()

            #import code
            #code.interact(local=dict(globals(), **locals()))

            #print(pipe_args[component])

            #component will never have both pipe and process in the same level. (?)
            if 'process' in pipe_args[component]:

                component_name = pipe_args[component]['process']
                n_use = pipeline_n_extract(self.n,self.n_ref,component_name)
                #print(n_use)
                #print(self.n_ref)
                #print(self.namespace[component_name])

                return self.namespace[component_name].invoke(params,n=n_use,n_ref=self.n_ref,pipe_args=pipe_args[component],upstream=upstream,\
                                                                       pipeID=self.pipeID,namespace = self.namespace,dag=self.dag,compdef=None)
            elif 'pipe' in pipe_args[component]:

                #test turn this off.
                #pipe_args,pipe_val,compdef,pipeID =pipetest_pargs_pval_compdef(self.pipe_args[component],self.namespace)

                #test possible bugfix
                pipe_args,pipe_val,compdef,pipeID =pipetest_pargs_pval_compdef(pipe_args[component],self.namespace)

                #although this didn't fix the current bug, it is probably more correct so leave it.
                #now, test saving a copy of params in the style of pipe_args above to avoid this persisting between components

                    #here  is where I'd test for the loop conditional

                if 'loop_on' in pipe_args: #and self.n==False: #not sure if this does anything..?

                    #check to see if there is an novr

                    #print(component)

                    #if component== "GetPE1_S1":
                    #    import code
                    #    code.interact(local=dict(globals(), **locals()))

                    #print(pipe_args['loop_on'])

                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                    if 'reset_loop' in pipe_args:
                        if pipe_args['reset_loop'] == 'y':
                            do_reset = True
                    else:
                        do_reset = False

                    #print(component)
                    #print(upstream)
                    #print(self.n_ref)
                    #print(self.n)

                    #if len(self.n)==2 and len(self.n_ref)==1:
                    #    import code
                    #    code.interact(local=dict(globals(), **locals()))
                    #print(self.n)
                    #print(self.n_ref)

                    return CombineExtLoop.invoke(params=params,component=pipe_val,pipe_args=pipe_args,pipeID=pipeID,\
                                                 namespace = self.namespace,compdef=compdef,process_peek = pipe_args['loop_on'],dag=self.dag,\
                                                 n=self.n, n_ref = self.n_ref, reset_loop = do_reset)
                else:

                    #reworking looping. Need to see what vars are available here to plug into pipeline_n_extract
                    #import code
                    #code.interact(local=dict(globals(), **locals()))

                    #assert 1==2
                    #print("THIS HAPPENED")

                    return pipe_val.invoke(params,n=self.n,n_ref=self.n_ref,pipe_args=pipe_args,upstream=upstream,\
                                           pipeID=pipeID,compdef=compdef,namespace = self.namespace,dag=self.dag)
        else:

            #import code
            #code.interact(local=locals())
            if component in self.namespace:
                #this means that there is a hardcoded process or pipeline as the component.
                #When hardcoding compdef, use a string to identify process or pipeline in namespace.
                #once you hardcode a component, you cannot go back to pipeargs (everything upstream also must be hardcoded
                self.namespace[component].invoke(self.params_copy,n=self.n,n_ref=self.n_ref,pipe_args={},upstream=upstream,\
                                               pipeID=pipeID,compdef=None,namespace = self.namespace,dag=self.dag)
            else:
                #this means that the component is not assigned. So, we return None. This can be done intentionally or unintentionally, be careful
                return None

            #what this also means, is that components cannot share namespace with pipelines/processes!

    def loop_component(self,component=None,final_process=False,upstream=[None],process_peek=None):

        #unsure if this is being used, but if so want to see if I can
        import code
        code.interact(local=dict(globals(), **locals()))
        if 'reset_loop' in pipe_args:
            if pipe_args['reset_loop'] == 'y':
                do_reset = True
        else:
            do_reset = False

        return CombineExtLoop.invoke(params=self.params_copy,component=component,pipe_args={},pipeID=self.pipeID,\
                                             namespace = self.namespace,compdef=None,process_peek = process_peek,dag=self.dag,\
                                     n=self.n,n_ref=self.n_ref,reset_loop = do_reset)

    @classmethod
    def invoke(cls,params,pipe_args={},upstream=[None],n='default',pipeID=None,namespace = None,compdef = None,dag=[],n_ref=None):

        #drop to pipeline name param level if present in keys.
        if cls.__name__ in params:
            params = param_smoosh(params,cls.__name__)

        #print(params["DLmodel*"])

        return cls(params,n,pipe_args,upstream,namespace,compdef,n_ref,dag=dag).run()

class INSTINCT_task(luigi.Task):

    ports = luigi.Parameter(significant=False) #these are the upstream ports , in order.

    ports_hash = luigi.ListParameter() #these are included to be hashed.

    #internal variable definitions
    hash_length = int(os.environ.get('HASHLEN'))

    hash_file = False #for processes which need to do this, specify this as true

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._Task__hash = int(str(self._Task__hash)[1:(1+self.hash_length)])

    def requires(self):
        if self.ports[0]!=None:
            for L in range(len(self.ports)):
                yield self.ports[L]

    def outfilegen(self):

        return self.outpath() + "/" + self.outfile

    def outpath(self):

        if self.ports[0]==None:
            #default to CacheRoot
            path = PARAMSET_GLOBALS['cache_root'] +'/'+ str(self._Task__hash)

        else:
            path = self.ports[0].outpath() +'/'+ str(self._Task__hash)

        if not os.path.exists(path): #might be redundant, make sure not slowing anything down
            os.makedirs(path)

        return path

    #convenience
    def getports(upstream):
        #print(upstream)
        #print(len(upstream))
        if upstream[0]!=None:
            ports = [upstream[L] for L in range(len(upstream))]
            ports_hash = [upstream[L]._Task__hash for L in range(len(upstream))]
        else:
            ports = [None]
            ports_hash = [None]
        return [ports,ports_hash]

class INSTINCT_process(INSTINCT_task):

    parameters = luigi.DictParameter(significant=False) #this is referenced within task
    param_string = luigi.Parameter() #this is only hashed

    param_string2 = luigi.Parameter(significant=False) #this is passed to run_cmd

    rerun_key = luigi.IntParameter() #all this does is change the hash to a unique value

    arguments = luigi.DictParameter(significant=False)
    descriptors = luigi.DictParameter(significant=False)

    processID = luigi.Parameter(significant=False)

    pipeID= luigi.Parameter(significant=False) #reference the pipeline that called the process

    def output(self):
        return luigi.LocalTarget(self.outfilegen())
    def run_cmd(self):
        #1st iteration of this, might modify

        methodjoin = '-'

        if self.descriptors['language']=='R':
            executable1 = 'Rscript '
            executable2 = '.R'
        elif self.descriptors['language']=='Python':
            executable1 = 'python '
            executable2 = '.py'
        elif self.descriptors['language']=='MATLAB':
            executable1 = ''
            executable2 = '.mat'
        elif self.descriptors['language']=='batch':
            executable1 = ''
            executable2 = '.bat'

        if self.descriptors['runtype']=='bin':
            executable2 = '.exe'
            methodjoin = '' #matlab does not allow for '-' in name
        #print(self.cmd_args)

        command0 = executable1 + PARAMSET_GLOBALS['project_root'] + 'lib/user/methods/' + self.processID +\
                       '/' + self.parameters['methodID'] + '/' + self.parameters['methodID'] + methodjoin + self.parameters['methodvers'] + executable2

        wrapper,wrapper_value = keyassess('wrapper',self.descriptors)
        if wrapper_value!='True':
            #import code
            #code.interact(local=locals())
            command2 = os.environ["INS_ARG_SEP"].join(self.cmd_args)

            command = command0 + ' '+ command2

        else:
            command1 = executable1 + PARAMSET_GLOBALS['project_root'] + 'lib/user/methods/' + self.processID + '/'+\
            self.processID  +"Wrapper" + executable2

            command2 = PARAMSET_GLOBALS['project_root'] + os.environ["INS_ARG_SEP"]+ os.environ["INS_ARG_SEP"].join(self.cmd_args)

            command = command1 + ' '+ command2

        if 'venv' in self.descriptors:
            print("******************************\nActivating virtual environment " + self.descriptors['venv_name'] + " with " + self.descriptors['venv'] + "\n******************************")
            if(self.descriptors['venv']=='Conda'):
                command_venv = self.descriptors['venv'] + ' activate ' + self.descriptors['venv_name'] + ' & '
                command = command_venv + command #append venv call to start of command. This assumes conda venv is set up to work on command line.

                pass_sub,pass_sub_val = keyassess('pass_sub_to_wrapper',self.descriptors)
                if pass_sub_val=='True':

                    command = command + os.environ["INS_ARG_SEP"]+ command0

                #import code
                #code.interact(local=dict(globals(), **locals()))
            else:
                print("VENV NOT YET CONFIGURED TO WORK ON BASE VENV")

        print("******************************\nRunning " + self.descriptors['language'] + " method " + self.parameters['methodID'] + methodjoin + self.parameters['methodvers'] +\
              " for process " + self.processID + "\n******************************")

        #temp:
        #print(command)

        print("******************************\nCommand params (can copy and paste): " + command2 +"\n******************************")

        #import code
        #code.interact(local=locals())
        subprocess.run(command,shell=True)

    #pipe_args is not refered to here since it is a placeholder to allow for dynamic calling between pipelines/processes
    @classmethod
    def invoke(cls,params,upstream=[None],n='default',pipe_args={},pipeID=None,namespace=None,dag=None,compdef=None,n_ref=None):

        #print(cls)
        #print(n)

        keytest,value = keyassess('drop',pipe_args)

        if keytest and value!=False:

            params = param_smoosh(params,value)

        paramssave = params

        processID = cls.__name__ #overwrite this if have to drop further to find params
        #print(cls.__name__)

        #if process name present, drop to process
        if cls.__name__ in params:
            params = params[cls.__name__] #always drop to processes available in params: control overrides in param file
        #if not present, try to use character matching.
        else:
            for f in range(len(list(params.keys()))):
                string_candidate = list(params.keys())[f]
                if '*' in string_candidate:

                    string_candidate_test = string_candidate[0:string_candidate.find('*')]

                    if string_candidate_test in cls.__name__:
                        params = params[string_candidate] #if candidate contains string and wildcard
                        processID = string_candidate_test
                        break
        #print(params)

        if 'parameters' in params:

            #if cls.__name__!="FormatGT":

            parameters = params['parameters'].copy()

            if len(parameters)>0:

                #if n==1:
                #import code
                #code.interact(local=locals())

                #extract methodID and method vers
                if params['descriptors']['runtype']!= "no_method":
                    methodparams = { k:v for k,v in parameters.items() if 'methodID' in k}
                    methodparams.update({ k:v for k,v in parameters.items() if 'methodvers' in k})

                    del parameters['methodID']
                    del parameters['methodvers']

                prm_tlist= sorted(parameters.items())

                prm_vals = [prm_tlist[x][1] for x in range(len(prm_tlist))]

                #import code
                #code.interact(local=dict(globals(), **locals()))

                if n != 'default':
                    #print('didit')
                    #print(n)
                    prm_vals_sort = [sorted(prm_vals[x])[n] if isinstance(prm_vals[x],list) else prm_vals[x] for x in range(len(prm_vals))]

                else:
                    #or resort the list so that it hashes in a standard way.
                    prm_vals_sort = [','.join(sorted(prm_vals[x])) if isinstance(prm_vals[x],list) else prm_vals[x] for x in range(len(prm_tlist))]

                #here, check for global parameters and inject in case of match
                for i in range(len(prm_vals_sort)):
                    while prm_vals_sort[i].startswith("[") and prm_vals_sort[i].endswith("]"): #attempt a bugfix

                        prm_vals_sort[i] = PARAMSET_GLOBALS['parameters'][prm_vals_sort[i][1:len(prm_vals_sort[i])-1]]
                    #also, check for leading and trailing whitespace
                    #prm_vals_sort[i] = prm_vals_sort[i].strip()

                #remove /n from strings if present
                for i in range(len(prm_vals_sort)):
                    if "\n" in prm_vals_sort[i]:
                        prm_vals_sort[i] = prm_vals_sort[i].replace('\n','')
                        #import code
                        #code.interact(local=dict(globals(), **locals()))

                param_string = ' '.join(prm_vals_sort)
                param_string2 = os.environ["INS_ARG_SEP"].join(prm_vals_sort)

                #if cls.__name__=="EventDetector":

                #append on the values for methodID and method to param string
                if params['descriptors']['runtype']!= "no_method":
                #squish together methodID and methodvers to be more consistent with existing R methods etc.
                    param_string = param_string + ' '+ methodparams['methodID'] + "-" + methodparams['methodvers']
                    param_string2 = param_string2 + os.environ["INS_ARG_SEP"]+ methodparams['methodID'] + "-" + methodparams['methodvers']

                    parameters.update(methodparams)

            else:
                param_string = ''
                param_string2 = ''

            #change the n value within parameters
            #if n!= 'default':
            #import code
            #code.interact(local=locals())
            keys = list(parameters.keys())
            any_list = False
            for i in range(len(keys)):
                if type(parameters[keys[i]])==list:
                    any_list=True
                    parameters_n = parameters.copy()
                    #print(n)
                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                    if n == 'default':
                        n_ = 0
                    else:
                        n_ = n
                    parameters_n[keys[i]]=sorted(parameters[keys[i]])[n_]
            if any_list==False:
                parameters_n = parameters
            #else:
                #parameters_n = parameters

        else:
            parameters_n = None
            param_string =""
            param_string2 =""

        #print(parameters_n)

        if 'arguments' in params:
            arguments = params['arguments'].copy()
            if 'splits' not in arguments:
                arguments.update({'splits':'1'})#

            #test for presence of global parameters and populate.
            arg_vlist= list(arguments.values())
            arg_klist= list(arguments.keys())
            for i in range(len(arg_vlist)):
                val = arg_vlist[i]
                if val.startswith("[") and val.endswith("]"):
                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                    arguments[arg_klist[i]] = PARAMSET_GLOBALS['parameters'][val[1:len(val)-1]]
        else:
            arguments = {'splits':'1'}
        if 'descriptors' in params:
            descriptors = params['descriptors'].copy()
        else:
            descriptors = None
        #get outpath and hash from ports

        #print(param_string)

        portsout = cls.getports(upstream)

        #if has an argument to rerun process, test for completion, and delete output if complete.
        #this is useful for tasks where the state of the precursor is intentionally not address (a db for example)
        #and use cases arise where remaining consistent is more important than using the most up to date info

        rerun_key = 0
        if 'rerun_key' in arguments:
            rerun_key = int(arguments['rerun_key'])

        process = cls(parameters=parameters_n,param_string=param_string,param_string2=param_string2,arguments=arguments,descriptors=descriptors,\
                   ports=portsout[0],ports_hash=portsout[1],processID=processID,pipeID=pipeID,rerun_key=rerun_key)

        #downstream will be set to empty to when just populating hashes for combextloop.
        if 'hard_rerun' in arguments:
            if arguments['hard_rerun']=='y' and dag!=None:
                #delete all of the downstream outputs
                dag._clean(process,'downstream')

        if 'trim_tree' in arguments:
            if arguments['trim_tree']=='y' and dag!=None:
                #delete all of the downstream outputs
                dag._clean(process,'upstream')

        return(process)

class INSTINCT_userprocess(INSTINCT_process):

    #instinct userprocess has the same run, but calls a specific instructions
    def get_modules_Art(self):

        from getglobals import PROJECT
        from art import getArt
        import random

        art = getArt(PROJECT,True,random.randint(1,10))

        return art

    def input_w_art(self,message):

        mlen = len(message)
        spaceslen = 40
        messagebuffer = " "* (spaceslen-round(mlen/2))

        out = input("\n " + r"""                  '.'.,'; '. .'.,',; ,' :','..'.,', '. ; ,'""" + "\n" +\
                    messagebuffer + message + "\n" + self.get_modules_Art() + "\n(y/n):")
        return out

    def get_userfile(self,only_file=False):

        file = self.outfile
        outf1 = file[0:file.find(".")]
        outf2 = file[self.outfile.find("."):]
        outmod = outf1 + self.userfile_append + outf2

        if only_file ==False:
            out =self.outpath() + "/" +outmod
        elif only_file==True:
            out = outmod
        return out

    def user_cancelation(self):
        os.environ["USERREVIEW"] = "False"
        os.environ["CANCELREVIEW"] = "True"
    def print_manifest(self):

        os.system("start " + self.outpath().replace('/','\\')) #windows friendly for copy paste (matters on NAS drive)

        text = 'INPUT: file named: "' + self.outfile + '" in opened directory: "' + self.outpath().replace('/','\\') + '"\n\n' +\
               'INFO:' + self.get_info()+ '"\n\n' +\
                "INSTRUCTIONS: " + self.parameters['instructions'] + '\n\nOUTPUT: save a new copy of the file in the opened directory as: "' + self.get_userfile(True) + '" \n'+\
                "When task is completed, enter y to Proceed, or n to abort." + "\n\n" +\
                "Proceed?"

        return text

    def file_modify(self,file): #if you want to modify the file prior to user working on it, override this in specific implementation
        print("")
        #for example, this could take a RAVENx file and add a column for analyst (based on user ID)
    def run(self):

        if os.environ.get('USERREVIEW')!="False":
            #this is the default value

            userreview = self.input_w_art("A userprocess task awaits! Perfom now?") #A userprocess task awaits! Perfom now?

            if userreview == 'y':
                os.environ["USERREVIEW"] = "True"

                #make a change: before calling this, move files into this task output folder- will be much more clear if running different versions, otherwise have already populated
                #files and folders which will be confusing to the user.

                shutil.copy(self.ports[0].outfilegen(), self.outfilegen())

                self.file_modify(self.outfilegen()) #in the specific usage of this class, you can change file_modify to prepare file for analysis.

                taskcomplete = self.input_w_art(self.print_manifest())

                if taskcomplete=='y':
                    #in this case, rename and move the file to correct output format.
                    #import code
                    #code.interact(local=dict(globals(), **locals()))

                    shutil.copy(self.get_userfile(), self.outfilegen())

                elif taskcomplete=='n':
                    self.user_cancelation()
                    os.remove(self.outfilegen())
                else:
                    raise ValueError("Invalid input to (y/n) prompt")

            elif userreview == 'n':
                self.user_cancelation() #this signifies when printing job output that a review was not performed
            else:
                raise ValueError("Invalid input to (y/n) prompt")
        #otherwise, do nothing and let the task fail
        #import code
        #code.interact(local=dict(globals(), **locals()))

        #idea: prompt user whether they want to perform tasks now, save preference as a temporary env variable if not.

class Split_process:

    #reason these are loaded in this way instead of appended to arguments is because arguments is a 'frozendict'
    splits = luigi.IntParameter()
    split_ID = luigi.IntParameter()
    process_ID = luigi.Parameter(significant=False)

    outpath_str = luigi.Parameter(significant=False)

    def outpath(self):
        return self.outpath_str
    def outfilegen(self,only_file=False):

        #import code
        #code.interact(local=locals())
        file = self.outfile
        outf1 = file[0:file.find(".")]
        outf2 = file[self.outfile.find("."):]
        outmod = outf1 + str(self.split_ID) + '_' + str(self.splits) + outf2
        #adding in splitnum here keeps names unique and does not cause partial outputs
        #to be interpreted as different size chunk between runs.
        if only_file ==False:
            out =self.outpath() + "/" +outmod #testing
        elif only_file==True:
            out = outmod
        return out

class SplitRun_process:
    def requires(self):
        return self.SplitInitial(parameters=self.parameters,param_string=self.param_string,param_string2=self.param_string2,arguments=self.arguments,\
                                 descriptors=self.descriptors,ports=self.ports,ports_hash=self.ports_hash,splits=self.splits,\
                                 split_ID=self.split_ID,process_ID=self.process_ID,outpath_str = self.outpath(),processID=self.processID,\
                                 pipeID=self.pipeID,rerun_key=self.rerun_key)

class Unify_process:

    def requires(self):
        for k in range(int(self.arguments['splits'])):
            yield self.SplitRun(parameters=self.parameters,param_string=self.param_string,param_string2=self.param_string2,arguments=self.arguments,\
                                descriptors=self.descriptors,ports=self.ports,ports_hash=self.ports_hash,splits=int(self.arguments['splits']),\
                                split_ID=k+1,process_ID=self.__class__.__name__,outpath_str = self.outpath(),processID=self.processID,\
                                pipeID=self.pipeID,rerun_key=self.rerun_key)
#pipelines:

#note: the looping function currently relies on some AFSC submodule file formats, and supporting functions, to correctly combine outputs.
class CombineExtLoop(INSTINCT_task):

    outfile = luigi.Parameter(significant=False)

    pipeID= luigi.Parameter(significant=False) #reference the pipeline that called the process

    def output(self):
        return luigi.LocalTarget(self.outfilegen())
    def run(self):

        if '.txt' in self.outfile: #this assumes it's going to be a raven format- can get more specific here
            #if you need other .txt formats.

            dataframes = [None] * len(self.ports)
            for L in range(len(self.ports)):
                dataframes[L] = pd.read_csv(self.ports[L].outfilegen(), delimiter = "\t") #
            Dat = pd.concat(dataframes,ignore_index=True)

            Dat["Selection"] = Dat.index + 1

            #import code
            #code.interact(local=locals())
            Dat.to_csv(self.outfilegen(),index=False, sep = "\t")
        elif self.outfile=="no_comb" and ('.tgz' in self.ports[0].outfilegen() or '.tar.gz' in self.ports[0].outfilegen()):

            os.mkdir(self.outfilegen())

            for L in range(len(self.ports)):
                outpath_name = self.outfilegen() +"/" + str(L+1) + "_" + os.path.basename(self.ports[L].outfilegen()[:self.ports[L].outfilegen().index(".")])
                os.mkdir(outpath_name)
                with tarfile.open(self.ports[L].outfilegen(), "r:gz") as tar:

                    #outdir =tarfile.open(self.outfile)
                    for member in tar:
                        if member.isdir():
                            continue
                        fname = member.name.rsplit('/',1)[1]
                        tar.makefile(member, outpath_name + '/' + fname)

                tar.close()

            #import code #unpack before combining into folder! Use commented out code at bottom of elifs
            #code.interact(local=dict(globals(), **locals()))

                    #tar.extract(outpath_name)

            #create a directory called no_comb with all of the files
            #os.mkdir(self.outfilegen())
            #for L in range(len(self.ports)):
            #    shutil.copy(self.ports[L].outfilegen(), self.outfilegen() +"/" + str(L+1) + "_" + self.ports[L].outfile)

        elif self.outfile=="no_comb":
            #create a directory called no_comb with all of the files
            os.mkdir(self.outfilegen())
            for L in range(len(self.ports)):
                shutil.copy(self.ports[L].outfilegen(), self.outfilegen() +"/" + str(L+1) + "_" + self.ports[L].outfile)

        elif 'FileGroupFormat.csv.gz' in self.outfile:

            #FG_dict = file_peek(self.ports[0].outfilegen(),fn_type = object,fp_type = object,st_type = object,dur_type = 'float64')

            dataframes = [None] * len(self.ports)
            for L in range(len(self.ports)):
                dataframes[L] = pd.read_csv(self.ports[L].outfilegen())#, dtype=FG_dict)
            Dat = pd.concat(dataframes,ignore_index=True)
            Dat['StartTime'] = pd.to_datetime(Dat['StartTime'])
            #import code
            #code.interact(local=locals())

            #Note: since I don't have access to the parameter to do this correctly, I should probably not do this at all.
            #however, do I know the implications if i don't do this here?
            Dat.drop(columns=['DiffTime'])

            Dat=get_difftime(Dat) #this is currently broken- difftime doesn't work between original format and format FG

            Dat.to_csv(self.outfilegen(),index=False)
        elif '.csv.gz' or '.csv.' in self.outfile:

            dataframes = [None] * len(self.ports)
            for L in range(len(self.ports)):
                dataframes[L] = pd.read_csv(self.ports[L].outfilegen()) #
            Dat = pd.concat(dataframes,ignore_index=True)

            Dat.to_csv(self.outfilegen(),index=False)
        #elif ".tgz" in self.outfile or ".tar.gz" in self.outfile:

            #pseudo: unpack all files into single directory, then tar that directory.

        #    import code
        #    code.interact(local=dict(globals(), **locals()))

        #    for L in range(len(self.ports)):
        #        outnamepath = self.outfile[:self.ports[L].outfilegen().index(".")]
        #        outdir =tarfile.open(self.outfile)

        #        outdir.extractall(outnamepath)

    @classmethod
    def invoke(cls,params,component,pipe_args={},upstream=[None],process_peek=None,novr='default',pipeID=None,namespace = None,compdef=None,dag=None,
               n_ref=None,n='default',reset_loop=False):
        #params in this case, just means a level above processes/subroutines

        #save these variables as copies so I don't modify self.n_ref

        #n = n.copy()
        #print(n)
        #print(isinstance(n, int))
        #print(n_ref)
        #print(process_peek)
        if n_ref!=None:
            n_ref = n_ref.copy()
        #import code
        #code.interact(local=locals())

        #if this is true, blank out n_ref and n
        if reset_loop:
            n_ref = None
            n = 'default'

        paramssave=params

        n_lens = []

        #make everything into lists
        if n_ref==None or isinstance(n_ref, str):
            n_ref= [n_ref]
            #n=[n]

        if isinstance(n, int):
            n = [n]

        if isinstance(process_peek, str):
            process_peek= [process_peek]

        for f in range(len(process_peek)):

            process_lookup = process_peek[f]

            parameters = params[process_lookup]["parameters"]

            n_len = None
            for i in parameters.keys():
                if type(parameters[i])==list:
                    n_len = len(parameters[i])

            if novr!= 'default':
                n_len = 1
                nval = novr
            else:
                nval = 0

            n_lens.append(range(n_len))

            #print(pipe_args)

        #print(process_peek)
        #print(n_lens)
        #print(n_ref)

        #might work a little better as a dict, but using the itertools below which seems to work better on lists.
        override_matches = set(process_peek) & set(n_ref)

        if len(override_matches)>0:
            #didthis=True
            override_matches_list = list(override_matches)

            for p in range(len(override_matches_list)):
                #new (process peek) will override old (n_ref).

                #find index of match in old
                del_ind = n_ref.index(override_matches_list[p])

                #print("DIDIT")

                n_ref = [x for i,x in enumerate(n_ref) if i!=del_ind]
                n = [x for i,x in enumerate(n) if i!=del_ind]

                #del n_ref[del_ind]
                #del n[del_ind]

                #print(n_ref)
                #print(n_ref != [None] and n_ref != [])
                #print(list(range(len(n))))

                override_matches = []
        #else:
        #    didthis = False

        #if didthis:
        #    print(n_lens)
        #    print(process_peek)

        if n_ref != [None] and n_ref != []:
            #if n_ref is not none need to append
            #append lists

            #if len(n_ref) !=len(n):
            #        import code
             #       code.interact(local=locals())

            for p in range(len(n)):
                n_lens.append(range(n[p],n[p]+1))
                process_peek.append(n_ref[p])

        #if didthis:
        #    print(n_lens)
        #    print(process_peek)

        #if is_list:
        if(len(process_peek)>1):

            import itertools
            perms = list(itertools.product(*n_lens))
            #convert tuples to list
            perms = [list(tup) for tup in perms]

            #assert len(perms[0]) == len(process_peek )
        else:

            perms =  n_lens[0]
            #import code
            #code.interact(local=locals())
        #else
        #    perms = n_lens[0]

        #print(perms)
        #print(process_peek)

        #print(n_ref)

        #[assert len(L)==len(process_peek) for L in perms]

        #here, determine if n_ref should be ignored (if absent), appended to (if unique) or overwritten (if the same).
        #also, see if I can access other parameters on the same level ('loop_reset') for instance.
        #if n_ref != None:

        #for p in n_ref:
        #print(perms)
        #print(process_peek)

        #print(perms)

        upstreamNew=[component.invoke(paramssave,n=L,n_ref=process_peek,pipe_args=pipe_args.copy(),upstream=upstream,pipeID=pipeID,namespace = namespace,\
                                      compdef=compdef,dag=dag) for L in perms]

        #this makes sure only unique inputs are retained. If not unique, why run it?
        upstreamNew = list(set(upstreamNew))
        n_len = len(upstreamNew)

        if n_len!=1:

            portsout = cls.getports(upstreamNew)

            noncombinable_formats = [".png","test",".tgz"]

            if any([(i in upstreamNew[0].outfile) for i in noncombinable_formats]):
                outmod = "no_comb"
            else:
                outmod = upstreamNew[0].outfile

            comb_process = cls(ports=portsout[0],ports_hash=portsout[1],outfile=outmod,pipeID=pipeID)

            return(comb_process)
        else:
            #if it isn't a loop, just return the component
            #import code
            #code.interact(local=locals())
            #print(perms)
            return component.invoke(paramssave,n=nval,pipe_args=pipe_args,upstream=upstream,pipeID=pipeID,namespace = namespace,compdef=compdef,dag=dag)

        #pass values into luigi task as parameters: outpath, hash, file type, etc.

class INSTINCT_job(INSTINCT_task):

    job_out=luigi.Parameter(significant=False) #means if output folder is changed, will keep hash but rerun
    #in new folder
    param_file_name=luigi.Parameter(significant=False) #for saving params. duplication helps with tracking.

    paramset_original=luigi.DictParameter(significant=False)

    pipenames=luigi.ListParameter(significant=False)

    def outpath(self):

        path = PARAMSET_GLOBALS['project_root'] +'Outputs/' + self.job_out + '/' + str(self._Task__hash)

        if not os.path.exists(path): #might be redundant, make sure not slowing anything down
            os.makedirs(path)

        return path

    def output(self):
        for n in range(len(self.ports)):
            filedest =self.outpath() + "/" + self.pipenames[n] + "_" + self.ports[n].outfile
            if '.gz' in filedest:
                filedest = filedest[0:-3]
            elif '.tgz' in filedest or '.tar.gz' in filedest:

                filedest = filedest[:filedest.index(".")]
            yield luigi.LocalTarget(filedest)

    def run(self):

        #find the outputs and dump them in common folder.

        nt.dump(self.paramset_original, self.outpath() +"/" +self.param_file_name +"_archive.nt")

        for n in range(len(self.ports)):

            if self.ports[n].outfile!="no_comb":

                #here, look for .gz and unzip before transfering! Make output not outfile but a non .gz modification of outfile.
                #import code
                #code.interact(local=locals())

                filepath = self.ports[n].outfilegen()

                filedest =self.outpath() + "/" + self.pipenames[n] + "_" + self.ports[n].outfile

                if '.gz' in filepath:
                    outnamepath = filedest[0:-3]
                    with gzip.open(filepath, 'rb') as f_in:
                        with open(outnamepath, 'wb') as f_out:
                            shutil.copyfileobj(f_in, f_out)
                elif '.tgz' in filepath or '.tar.gz' in filepath:

                    outnamepath = filedest[:filedest.index(".")]
                    outdir =tarfile.open(filepath)

                    outdir.extractall(outnamepath)

                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                else:
                    shutil.copy(filepath, filedest)

            else:

                shutil.copytree(self.ports[n].outfilegen(), self.outpath() + "/" + self.pipenames[n] + "_" +  self.ports[n].outfile)

                #import code
                #code.interact(local=locals())

            #add in functionality here to unzip files in outputs!

    @classmethod
    def invoke(cls,job,paramset_original,param_file_name,pipenames,pipeID=None): #testing out change- params was 2nd arg here

        #import code
        #code.interact(local=locals())

        job_out=list(job.keys())[0]
        upstream=list(job.values())[0]

        #print(upstream[0].outpath())

        portsout = cls.getports(upstream)

        return(cls(ports=portsout[0],ports_hash=portsout[1],job_out=job_out,pipenames=pipenames,\
                   paramset_original=paramset_original,param_file_name=param_file_name))

class INSTINCT_job_wrapper(luigi.WrapperTask,INSTINCT_job): #may not work, as both inherit from luigi.task...
    pass

##############################################


from Helpers.Artist import *
from Helpers.DatabaseReceiver import *
import sqlite3

# Author: Richard Jongenburger

class Timer:

    # Array with images of the numbers.
    numbers = []

    # Array with the images of the small numbers.
    numbers_small = []

    # Array that represents the time.
    time = []

    # Array that represents the best time.
    best_time = []

    # Array with boolean values to test if the numbers can be drawn.
    # They can't be drawn when the number is zero.
    can_draw = []

    # Event id.
    MILLISEC_PASSED_EVENT = pygame.USEREVENT + 1

    # Get images from database.
    clock = DatabaseReceiver.get_number_img("Klok", "Klok")
    no_time_line = DatabaseReceiver.get_number_img("No_time", "No_time")

    # The x coordinate the best_time and timer should start to draw.
    timer_start_x = 820

    def __init__(self):
        # Make an array with the hundreds, tens, ones and tenths.
        # Every number should be zero at the start of the level.
        self.time = [0, 0, 0, 0]

        # This array is to determine if the number can be drawn on the
        # surface.
        # Because we only want to draw the number
        # if it is > 0. To make it look better.
        self.can_draw = [False, False, False, True]

        # Get all the images of the numbers
        # from the database and put it in the numbers array.
        for x in range(0, 10):
            self.numbers.append(DatabaseReceiver.get_number_img
                                ("Big", "hud_" + str(x)))

        # Get all the images of the small numbers
        # from the database and put it in the numbers_small array.
        for x in range(0, 10):
            self.numbers_small.append(DatabaseReceiver.get_number_img
                                      ("Small", "hud_" + str(x)))

        # Set a timer by using the set_timer method from the
        # pygame library.
        # The first number is the event id. Every event in
        # pygame has an id.
        # Because we want to make our own event,
        # we have to define an id ourself.
        #
        # Pygame's documatation said the following :
        # It is best to use the value between
        # pygame.USEREVENT and pygame.NUMEVENTS.
        # So we use pygame.USEREVENTS + 1.
        # The second argument is the number of milliseconds should pass.
        #
        # So basically this gives every 100 milliseconds
        # an event back with id: pygame.USEREVENTS + 1.
        pygame.time.set_timer(self.MILLISEC_PASSED_EVENT + 1, 100)

    def update(self):
        # Get all the pygame events by using pygame.event.get().

        for event in pygame.event.get():
            # Check with event.type if it is our event we made in
            # the init.
            # (pygame.USEREVENT + 1)
            if event.type == self.MILLISEC_PASSED_EVENT + 1:
                # Increase the 4th number with 1.
                # So that every 100 milliseconds the
                # 4th number will increase by 1. Like a timer.
                self.time[3] += 1

        # Fourth number:
        # If the 4th number is bigger then 9. (so it's ten)
        if self.time[3] > 9:
            # Increase the 3th number by one.
            self.time[2] += 1

            # Set the 4th number back to zero.
            self.time[3] = 0

            # And now that the 3th number is higher than zero,
            # we can draw it.
            self.can_draw[2] = True

        # Thirth number:
        # If the 3th number is bigger then 9. (so it's ten)
        if self.time[2] > 9:

            # Increase the 2nd number by one.
            self.time[1] += 1

            # Set the 3th number back to zero.
            self.time[2] = 0

            # And now that the 3th number is higher than zero,
            # we can draw it.
            self.can_draw[1] = True

        # Second number:
        # If the 2th number is bigger then 9. (so it's ten)
        if self.time[1] > 9:

            # Increase the 1st number by one.
            self.time[0] += 1

            # Set the 2nd number back to zero.
            self.time[1] = 0

            # And now that the 3th number is higher than zero,
            # we can draw it.
            self.can_draw[0] = True

        # First number:
        # If the 1th number is bigger then 9. (so it's ten)
        if self.time[0] > 9:
            # Reset the time. So the time will reset when its over 999.
            self.reset_time()

    # Reset all the numbers to zero.
    def reset_time(self):
        self.time[0] = 0
        self.time[1] = 0
        self.time[2] = 0
        self.time[3] = 0

    # Draw every object on the surface.
    def draw(self):
        self.draw_clock()
        self.draw_timer()
        self.draw_best_time()

    # Draw the gray clock.
    def draw_clock(self):
        Artist.draw_textures(self.clock, (770, 50))

    def draw_timer(self):
        # The x coordinate the image should start to draw.
        x = self.timer_start_x

        for i in range(0, 4):
            if self.can_draw[i]:
                # If it's the last number (tenths),
                # it should be drawn with less space and the y value
                # higher.
                if i == 3:
                    # Set the space between the 3th and 4th number.
                    x += 6

                    # Draw the last number.
                    Artist.draw_textures(self.numbers_small[self.time[i]],
                                         (x, 75))
                    break
                # Draw a number.
                Artist.draw_textures(self.numbers[self.time[i]],
                                     (x, 60))

            # Set the space between the images.
            x += 30

    # Calculate the best time and put it in the database.
    def save_best_time(self, level):
        # Multiple every number times what they represent.
        # The first number belongs to the 'hundreds',
        # so we do it times 100 etc.
        # We need to calculate the current time and the current
        # best time.
        time_temp = self.time[0] * 100 + self.time[1] * 10 + self.time[2] + (
            self.time[3] * 0.1)
        best_time_temp = self.best_time[0] * 100 + self.best_time[1] * 10 + (
            self.best_time[2] + self.best_time[3] * 0.1)

        # If the time is less then the current best time.
        # Put it in the database.
        # So you actually are faster then the current.
        #
        # Also put it in the database when the best_time is zero.
        # So it also saves when it's the first time you completed
        # the level.
        if time_temp < best_time_temp or best_time_temp == 0:
            # Save the time in the database.
            DatabaseReceiver.save_timer(str(self.time[0]), str(self.time[1]),
                                        str(self.time[2]), str(self.time[3]),
                                        str(level))

    # Get the best time from database and put it in the best_time array.
    def load_best_time(self, level):
        self.best_time = DatabaseReceiver.load_timer(str(level))

    def draw_best_time(self):
        # The x coordinate the image should start to draw.
        x = self.timer_start_x

        # If the best time is zero (so there isn't a beset time yet),
        # draw an image of a dash.
        if self.best_time[0] == 0 and self.best_time[1] == 0 \
                and self.best_time[2] == 0 and self.best_time[3] == 0:
            Artist.draw_textures(self.no_time_line, (x + 50, 700))

        for i in range(0, 4):
            # If it's the last number (tenths),
            # it should be drawn with less space and the y value higher.
            if i == 3:
                # Set the space between the 3th and 4th number.
                x += 6

                # Draw the last number.
                # Only when the number isn't zero.
                # So we don't see the numbers before it.
                if self.best_time[i] != 0:
                    Artist.draw_textures(self.numbers_small[self.best_time[i]],
                                         (x, 715))
                break

            # Draw the first three numbers.
            # Only when the number isn't zero too.
            if self.best_time[i] != 0:
                Artist.draw_textures(self.numbers[self.best_time[i]], (x, 700))

            # Set the space between the images.
            x += 30

    # Rest the best time in the database and on the game's surface.
    def reset_best_time(self, level):
        DatabaseReceiver.reset_timer()
        self.load_best_time(level)


"""
<Implementation>
<BFS>
"""
from sys import *
from collections import deque

dx = [1, 0, -1, 0]
dy = [0, 1, 0, -1]

# INPUT
N, M = map(int, stdin.readline().rstrip().split())
BOARD = [list(stdin.readline().rstrip()) for _ in range(N)]
VISIT = [[[[False] * M for _ in range(N)] for _ in range(M)] for _ in range(N)]

def move(x, y, dx, dy):
    dd = 0
    while BOARD[x][y] != "O" and BOARD[x + dx][y + dy] != "#":
        x += dx
        y += dy
        dd += 1

    return x, y, dd

def bfs():
    QUEUE = deque()  # queue

    # Red, Blue
    for i in range(N):
        for j in range(M):
            if BOARD[i][j] == "R":
                rx, ry = i, j
            elif BOARD[i][j] == "B":
                bx, by = i, j

    # start
    QUEUE.append([rx, ry, bx, by, 1])
    VISIT[rx][ry][bx][by] = True

    # BFS
    while QUEUE:
        rx, ry, bx, by, d = QUEUE.popleft()

        if d > 10:
            return -1
        for i in range(4):
            nrx, nry, rd = move(rx, ry, dx[i], dy[i])
            nbx, nby, bd = move(bx, by, dx[i], dy[i])

            if BOARD[nbx][nby] != "O":
                if BOARD[nrx][nry] == "O":
                    return d

                if nrx == nbx and nry == nby:
                    if rd > bd:
                        nrx -= dx[i]
                        nry -= dy[i]
                    else:
                        nbx -= dx[i]
                        nby -= dy[i]

                if not VISIT[nrx][nry][nbx][nby]:
                    VISIT[nrx][nry][nbx][nby] = True
                    QUEUE.append([nrx, nry, nbx, nby, d + 1])
    return -1

print(bfs())


import yaml
from io import StringIO
import pandas as pd

import logging
logger = logging.getLogger('em.psse')

import os
dirname = os.path.dirname(__file__)
with open('{}/psse-modes.yaml'.format(dirname),'r') as in_file:
	modes = yaml.load(in_file)

def get_signals(line_num,line,current_mode):
	#print(line_num,line)
	signals = []
	for m in modes:
		for s in m['signal']:
			if 'text' in s:
				if s['text'] in line:
					signals.append((s,m))
			if 'line' in s:
				if s['line'] == line_num:
					signals.append((s,m))
	return signals

def read_transformer(lines,records):
	iter_lines = iter(lines)
	#records=[]
	headers = [r['header'] for r in records]
	line_holder={
		2: [
			[headers[0]],
			[headers[1]],
			[headers[2]],
			[headers[3]]
		],
		3: [
			[headers[0]],
			[headers[1]],
			[headers[2]],
			[headers[3]],
			[headers[4]]
		]
	}
	for line in iter_lines:
		rows=[line]
		rows.append(next(iter_lines))
		rows.append(next(iter_lines))
		rows.append(next(iter_lines))
		windings = None
		try:
			flag=int(line.split(',')[2])
			if flag != 0:
				raise ValueError('Not a 2 phase winding')
			windings = 2
		except Exception as e:
			windings = 3
			rows.append(next(iter_lines))

		for r in range(len(rows)):
			line_holder[windings][r].append(rows[r])

	dfs={
		2:[],
		3:[]
	}
	for winding in line_holder:
		rc=0
		for record in line_holder[winding]:
			rc+=1
			text = StringIO(''.join(record))
			dfs[winding].append(pd.read_table(text,sep=',', error_bad_lines=False))
			logger.debug("{} {} {}".format(winding,rc,len(record)))
			logger.debug("{}".format(record[0]))
	return dfs

def read_twodc(lines,records):
	iter_lines = iter(lines)
	#records=[]
	headers = [r['header'] for r in records]
	line_holder=[
		[headers[0]],
		[headers[1]],
		[headers[2]]
	]
	for line in iter_lines:
		rows=[line]
		rows.append(next(iter_lines))
		rows.append(next(iter_lines))

		for r in range(len(rows)):
			line_holder[r].append(rows[r])

	dfs=[]
	rc=0
	for record in line_holder:
		rc+=1
		text = StringIO(''.join(record))
		dfs.append(pd.read_table(text,sep=','))
		logger.debug("{} {}".format(rc,len(record)))
		logger.debug("{}".format(record[0]))

	return dfs

def parse_raw(in_file_name):
	"""
	This function will parse a RAW file and return a PyPSA model
	"""

	# Initialize output
	output ={}
	for item in modes:
		key=item['key']
		output[key]={
			"name":item['name'],
			"key":key,
			"lines":[]
		}
		# Set up column structure
		if 'columns' in item:
			output[key]['columns']=item['columns']
			header = ",".join([c['name'] for c in item['columns']]).replace(' ','')+'\n'
			output[key]['header']=header
			output[key]['lines'].append(header)
		if 'records' in item:
			output[key]['records'] = item['records']
			for record in output[key]['records']:
				header = ",".join([c['name'] for c in record['columns']]).replace(' ','')+'\n'
				record['header']=header
		# Get parsing info
		output[key]['parse']=item.get('parse',{})

	# Initialize header mode
	current_mode = {
		'key': 'header'
	}

	# Read file and store in container
	with open(in_file_name,'r') as in_file:
		line = in_file.readline()
		line_num = 0
		while line:
			line_num+=1

			START = None
			STOP = None

			# Process signals
			signals = get_signals(line_num,line,current_mode)
			for signal,mode in signals:
				logger.debug("Signal: {} {} {}".format(line_num,signal['command'],mode['key']))
				if signal['command'] == 'start':
					START = mode
				if signal['command'] == 'stop':
					STOP = mode

			if current_mode and START and not STOP:
				raise ValueError('Current mode was never stopped')

			# Store lines
			if current_mode and (not STOP or 'keep_tail' in STOP):
				key = current_mode['key']
				output[key]['lines'].append(line)
				#print key

			if STOP and current_mode and current_mode['key'] != STOP['key']:
				raise ValueError('Attempting to stop a different mode',current_mode['key'],STOP['key'])
			elif START:
				current_mode = START
			elif STOP:
				current_mode = None

			line = in_file.readline()

	logger.debug("Captured Lines")
	for i in output:
		logger.debug('Item: {}, length: {}'.format(i,len(output[i]['lines'])))

	for i in output:
		if 'lines' not in output[i]:
			logger.debug('no lines {}'.format(i))
			continue
		lines=output[i]['lines']
		if len(lines)==1:
			logger.debug('only header {}'.format(i))
			continue

		if 'read_table' in output[i]['parse']:
			text = StringIO(''.join(lines))
			output[i]['df'] = pd.read_table(text,sep=',')

		if 'read_transformer' in output[i]['parse']:
			output[i]['dfs']=read_transformer(lines,output[i]['records'])
			df2=pd.concat(output[i]['dfs'][2],axis=1, sort=False)
			df3=pd.concat(output[i]['dfs'][3],axis=1, sort=False)
			output[i]['df']=df3.append(df2,sort=False)

		if 'read_twodc' in output[i]['parse']:
			output[i]['dfs']=read_twodc(lines,output[i]['records'])
			output[i]['df']=pd.concat(output[i]['dfs'],axis=1, sort=False)

		logger.debug('{} {} {}'.format(i, len(lines), 'df' in output[i]))
		if 'df' in output[i]:
			logger.info('Parsed {} {}'.format(len(output[i]['df']),i))
			if len(lines)>0:
				logger.debug("{}".format(lines[0]))

	return output


from django.http import HttpResponse, JsonResponse
from django.utils.translation import gettext_lazy as _

from dalite.views.errors import response_400
from dalite.views.utils import get_json_params
from peerinst.models import Teacher

from ..logger import logger
from ..models import Reputation
from .decorators import logged_in_non_student_required

@logged_in_non_student_required
def teacher_reputation(req):
    args = get_json_params(req, args=["id"])
    if isinstance(args, HttpResponse):
        return args
    (teacher_id,), __ = args

    try:
        teacher = Teacher.objects.get(pk=teacher_id)
    except (Teacher.DoesNotExist, AttributeError):
        return response_400(
            req,
            msg=_("This teacher doesn't exist."),
            logger_msg=(f"Tried to obtain teacher with pk {teacher_id}"),
            log=logger.warning,
        )

    if teacher.reputation is None:
        teacher.reputation = Reputation.create(teacher)
        teacher.save()

    reputation, reputations = teacher.reputation.evaluate()

    data = {"reputation": reputation, "reputations": reputations}

    return JsonResponse(data, status=200)


"""
* 가장 긴 동일 값의 경로
동일한 값을 지닌 가장 긴 경로를 찾아라.
- Example 1
Input :
          5
         / \
        4   5
       / \   \
      1   1   5
Output : 2
Explaination : 루트에서 오른쪽 노드 끝까지 5->5->5로 가장 긴 이동 거리가 2이다.
- Example 2
Input :
          1
         / \
        4   5
       / \   \
      4   4   5
Output : 2
Explaination : 왼쪽 리프 노드 4에서 형제 노드 4까지 4->4->4로 가장 긴 이동 거리가 2이다.
"""

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    result: int = 0
    def longestUnivaluePath(self, root: TreeNode) -> int:
        def dfs(node: TreeNode) -> int:
            if node is None:
                return 0

            left = dfs(node.left)
            right = dfs(node.right)

            if node.left and node.left.val == node.val:
                left += 1
            else:
                left = 0
            if node.right and node.right.val == node.val:
                right += 1
            else:
                right = 0

            # 왼쪽과 오른쪽 자식 노드 간 거리의 합 최댓값이 결과
            self.result = max(self.result, left + right)
            # 자식 노드 상태값 중 큰 값 리턴
            return max(left, right)

        dfs(root)
        return self.result

if __name__ == '__main__':
    solution = Solution()
    print(solution.longestUnivaluePath(TreeNode(5, TreeNode(4, TreeNode(1), TreeNode(1)), TreeNode(5, None, TreeNode(5)))))


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 17 15:43:56 2018

@author: Administrator

"""
from  matplotlib  import pyplot as plt
import numpy as np
import pymc3 as pm
import scipy as sp
import seaborn as sns
from theano import tensor as tt
import pandas as pd

K=30

M=2
e=np.ones((11,1))
I=np.eye(11)

omega=np.empty_like(e)#11,1
omega[0]=sp.stats.norm(loc=0.,scale=sigmaw)
for i in range(1,11):
    omega[i]=sp.stats.norm(loc=omega[i-1.]*rho,scale=sigmaw*sp.sqrt(1.-rho))

def pdf(omega):
    C=np.zeros((11,1))
    for i in range(0,11):
        C[i]=omega[i].pdf()
    return C[:,0].cumprod(axis=0)

with pm.Model() as model:
    mu=pm.Normal('mu',mu=0.,sd=0.1)
    xi=pm.InverseGamma('xi',1.,1.0)
    b=pm.Normal('b',mu=0.,sd=xi)
    sigam=pm.Uniform('sigma',0.,0.2)

    y=pm.MvNormal('y',mu=(mu+b)*e+omega,cov=sigma**2.*I)


"""
File that defined all the serializer used in our API

See http://www.django-rest-framework.org/api-guide/serializers/
See http://www.django-rest-framework.org/api-guide/fields/
See http://www.django-rest-framework.org/api-guide/relations/
"""
from rest_framework import serializers
from project.models import *
from django.contrib.auth.models import User, Group

class QwirkGroupSerializer(serializers.ModelSerializer):

	class Meta:
		model = QwirkGroup
		fields = ('id', 'name', 'isPrivate', 'isContactGroup')

class UserSerializer(serializers.ModelSerializer):

	class Meta:
		model = User
		fields = ('username', 'email', 'password', 'first_name', 'last_name')
		extra_kwargs = {
			'password': {'write_only': True}
		}

class QwirkUserSerializerSimple(serializers.ModelSerializer):
	user = UserSerializer(read_only=True)

	class Meta:
		model = QwirkUser
		fields = ('user', 'bio', 'birthDate', 'status', 'avatar')
		depth = 1

class MessageSerializer(serializers.ModelSerializer):
	qwirkGroup = QwirkGroupSerializer(read_only=True)
	qwirkUser = QwirkUserSerializerSimple(read_only=True)

	class Meta:
		model = Message
		fields = ('qwirkUser', 'qwirkGroup', 'text', 'dateTime', 'type', 'file')

class MessageSerializerSimple(serializers.ModelSerializer):
	qwirkGroup = serializers.SlugRelatedField(
		read_only=True,
		slug_field='name'
	)

	qwirkUser = serializers.StringRelatedField(read_only=True)

	class Meta:
		model = Message
		fields = ('qwirkGroup', 'qwirkUser', 'text', 'dateTime', 'type', 'file')

class NotificationSerializer(serializers.ModelSerializer):
	qwirkUser = QwirkUserSerializerSimple(read_only=True)
	message = MessageSerializer(read_only=True)

	class Meta:
		model = Notification
		fields = ('message', 'qwirkUser', 'dateRead')

class NotificationSerializerSimple(serializers.ModelSerializer):
	message = MessageSerializerSimple(read_only=True)

	class Meta:
		model = Notification
		fields = ('message', 'dateRead')

class ContactSerializer(serializers.ModelSerializer):
	qwirkUser = QwirkUserSerializerSimple(read_only=True)
	qwirkGroup = QwirkGroupSerializer(read_only=True)

	class Meta:
		model = Contact
		fields = ('qwirkUser', 'status', 'qwirkGroup')

class QwirkUserSerializer(serializers.ModelSerializer):
	qwirkGroups = QwirkGroupSerializer(many=True, read_only=True)
	contacts = ContactSerializer(many=True, read_only=True)
	user = UserSerializer(read_only=True)
	notifications = NotificationSerializerSimple(many=True, read_only=True)

	class Meta:
		model = QwirkUser
		fields = ('notifications', 'user', 'bio', 'birthDate', 'qwirkGroups', 'contacts', 'status', 'avatar')


import cv2
import logging
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np

def vis_pose(image_url, affine, max_idx, max_val, kp_names, skeletons):
    # subplots with 3 columns
    fig, (ax1, ax2, ax3) = plt.subplots(1, 3)
    fig.set_size_inches(8, 6)
    fig.suptitle("Keypoints for {}".format(image_url), fontsize="x-large")

    # column 1
    img = cv2.imread(image_url)
    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    img1 = cv2.resize(img, (256, 512), interpolation=cv2.INTER_LINEAR)
    ax1.imshow(img1)
    ax1.axis("off")

    # dotted image for column 2 and 3
    inv_affine = np.linalg.inv(affine)
    affine_pts = np.ones((len(max_idx), 3))
    affine_pts[:, :2] = np.array(max_idx) * 4.0
    pts = np.dot(inv_affine, affine_pts.T).T[:, :2]
    img_dotted = (0.5 * img).astype(np.uint8)

    # column 2
    img2 = img_dotted.copy()
    for p, name in zip(pts, kp_names):
        cv2.circle(img2, (int(p[0]), int(p[1])), 8, (0, 128, 255),
                   thickness=-1, lineType=cv2.FILLED)
        cv2.putText(img2, "{}".format(name), (int(p[0]) + 10, int(p[1]) + 10),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 128), 1, lineType=cv2.LINE_AA)
    img2 = cv2.resize(img2, (256, 512), interpolation=cv2.INTER_LINEAR)
    ax2.imshow(img2)
    ax2.axis("off")

    # column 3
    img3 = img_dotted.copy()
    for p in pts:
        cv2.circle(img3, (int(p[0]), int(p[1])), 8, (0, 128, 255),
                   thickness=-1, lineType=cv2.FILLED)
    for (i, j) in skeletons:
        pt_a = pts[i]
        pt_b = pts[j]
        cv2.line(img3, (int(pt_a[0]), int(pt_a[1])),
                 (int(pt_b[0]), int(pt_b[1])), (255, 255, 128), 2)
    img3 = cv2.resize(img3, (256, 512), interpolation=cv2.INTER_LINEAR)
    ax3.imshow(img3)
    ax3.axis("off")

    # adjust margins
    plt.subplots_adjust(top=1, bottom=0.01, right=0.98, left=0.02, hspace=0, wspace=0.05)
    plt.margins(0, 0)

    # save figure
    save_url = image_url.split(".")[0] + "_pose.jpg"
    plt.savefig(save_url)
    logging.info("Visualization in {}".format(save_url))


import os
import shutil
import random
import scipy.io

def move(dir_A, dir_B, percent):
    # 如果目录 B 不存在，则创建目录 B
    if not os.path.exists(dir_B):
        os.makedirs(dir_B)

    # 获取目录 A 中所有 .wav 文件的路径
    wav_files = [f for f in os.listdir(dir_A) if f.endswith('.WAV')]

    # 计算要提取的文件数
    num_files_to_extract = int(len(wav_files) * percent)

    # 随机选择要提取的文件
    files_to_extract = random.sample(wav_files, num_files_to_extract)

    # 将选定的文件移动到目录 B
    for file_name in files_to_extract:
        file_path = os.path.join(dir_A, file_name)
        dest_path = os.path.join(dir_B, file_name)
        shutil.move(file_path, dest_path)
    return True

def split_txt(input_path, part):
    with open(input_path, 'r') as input_file:
        lines = input_file.readlines()
        num_lines = len(lines)
        chunk_size = num_lines / part
        name = -15
        for i in range(part):
            start_index = i * chunk_size
            end_index = (i + 1) * chunk_size
            if i == part - 1:
                end_index = num_lines
            chunk = lines[start_index:end_index]
            name += 5
            with open(f'/ddnstor/imu_panjiahui/dataset/SHdomainNew/loader_txt/test_clean_snr{name}_list.txt',
                      'w') as output_file:
                output_file.writelines(chunk)

def gen_wav_scp(file_dir, txt_path):
    file_paths = []
    for root, dirs, files in os.walk(file_dir):
        for file in files:
            file_path = os.path.join(root, file)
            file_paths.append(file_path)

    with open(txt_path, "w") as f:
        for file_path in file_paths:
            f.write(file_path + "\n")

def compare_folders(folder1, folder2, folder3):
    files1 = set(os.listdir(folder1))
    files2 = set(os.listdir(folder2))
    files3 = set(os.listdir(folder3))

    # 找到不一致的文件
    diff_files = (files1 ^ files2) | (files1 ^ files3) | (files2 ^ files3)

    for file in diff_files:
        file_path1 = os.path.join(folder1, file)
        file_path2 = os.path.join(folder2, file)
        file_path3 = os.path.join(folder3, file)

        # 删除不一致的文件
        if os.path.exists(file_path1):
            print(file_path1)
            os.remove(file_path1)
        if os.path.exists(file_path2):
            print(file_path2)
            os.remove(file_path2)
        if os.path.exists(file_path3):
            print(file_path3)
            os.remove(file_path3)
    return True

def merge_files(folder_path, output_file):
    rir_list = ['1.0', '0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9']
    snr_list = ['-10', '-5', '0', '5', '10']

    merged_content = ""
    for snr in snr_list:
        for rir in rir_list:
            txt_path = folder_path + 'wav_scp_test_' + rir + '_snr' + snr + '.txt'
            if os.path.isfile(txt_path):
                with open(txt_path, "r") as file:
                    file_content = file.read()
                    merged_content += file_content
    with open(output_file, "w") as merged_file:
        merged_file.write(merged_content)
    return True

if __name__ == '__main__':
    # 定义目录 A 和目录 B 的路径
    # dir_A = '/ddnstor/imu_panjiahui/dataset/SHdomain/TIMIT/Data/train_val_clean/'
    # dir_B = '/ddnstor/imu_panjiahui/dataset/SHdomain/TIMIT/Data/val_clean/'

    # 1、按照比例移动A的内容到B中
    # 定义要提取的文件比例（这里是 10%）
    # percent = 0.1
    # move(dir_A, dir_B, percent)

    # 2、移动目录A 到目录B中
    # 使用 shutil.move() 函数将目录 A 移动到目录 B 中
    # dir_A = '/home/imu_panjiahui/SHdomain/SHC_baseline/data_prepare/MIC_data/'
    # dir_B = '/ddnstor/imu_panjiahui/dataset/SHdomain/'
    # shutil.move(dir_A, dir_B)

    # 3、A.txt里的内容分为5部分 分别写入五个.txt文件中
    # input_path = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/loader_txt/test_clean_list.txt'
    # part = 5
    # split_txt(input_path, part)

    # # 4、把生成的数据路径写入wav_scp
    # file_dir = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/train/mix'
    # txt_path = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/loader_txt/wav_scp_train.txt'
    # gen_wav_scp(file_dir, txt_path)

    # 5、比较并删除不一致的文件
    # folderA = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/train/mix'
    # folderB = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/train/noreverb_ref'
    # folderC = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/train/reverb_ref'
    # compare_folders(folderA, folderB, folderC)

    # rir_list = ['1.0', '0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9']
    # snr_list = ['-10', '-5', '0', '5', '10']
    # for snr in snr_list:
    #     for rir in rir_list:
    #         folderA = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/test/test_' + rir + '_snr' + snr + '/mix'
    #         folderB = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/test/test_' + rir + '_snr' + snr + '/noreverb_ref'
    #         folderC = '/ddnstor/imu_panjiahui/dataset/SHdomainNew/TIMIT/generated_data_cir_uniform_16/test/test_' + rir + '_snr' + snr + '/reverb_ref'
    #         compare_folders(folderA, folderB, folderC)
    #
    #         txt_path = '/data01/data_pjh/SHdomainNew/loader_txt/wav_scp_test_' + rir + '_snr' + snr + '.txt'
    #         gen_wav_scp(folderA, txt_path)

    # 6、将文件夹内的 几个txt 文件的内容合并到一个新的 merged.txt 文件中：
    # folder_path = '/data01/data_pjh/SHdomainNew/loader_txt/'
    # output_file = '/data01/data_pjh/SHdomainNew/loader_txt/wav_scp_test.txt'
    # merge_files(folder_path, output_file)

    # 7、加载.mat文件并输出
    # test_list = [
    #     'test_0.2_snr-10', 'test_0.2_snr-5', 'test_0.2_snr0', 'test_0.2_snr5', 'test_0.2_snr10',
    #     'test_0.6_snr-10', 'test_0.6_snr-5', 'test_0.6_snr0', 'test_0.6_snr5', 'test_0.6_snr10',
    #     'test_1.0_snr-10', 'test_1.0_snr-5', 'test_1.0_snr0', 'test_1.0_snr5', 'test_1.0_snr10',
    # ]
    #
    # for test_name in test_list:
    #     gcrnpath = '/home/imu_panjiahui/MixFormer/GCRN_baseline/model_gcrn_baseline/result/' + test_name
    #     dpcrnpath = '/home/imu_panjiahui/MixFormer/DPCRN_baseline/model_dpcrn/result/' + test_name
    #     mixformerpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer/model_dpcrn_mixformer/result/' + test_name
    #     depthwisefirstpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer/model_dpcrn_mixformer_noattention_depthfirst/result/' + test_name
    #     noattentionpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer/model_dpcrn_mixformer_noattention/result/' + test_name
    #     nodprnnpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer/model_dpcrn_mixformer_nodprnn/result/' + test_name
    #
    #     gcrn_base = scipy.io.loadmat(gcrnpath + '_metrics.mat')
    #     dpcrn_base = scipy.io.loadmat(dpcrnpath + '_metrics.mat')
    #     mixformer_pro = scipy.io.loadmat(mixformerpath + '_metrics.mat')
    #     depthwisefirst_pro = scipy.io.loadmat(depthwisefirstpath + '_metrics.mat')
    #     noattention_pro = scipy.io.loadmat(noattentionpath + '_metrics.mat')
    #     nodprnn_pro = scipy.io.loadmat(nodprnnpath + '_metrics.mat')
    #
    #     print(test_name + " PESQ:")
    #     print(
    #         'mix:' + "{:.2f}".format(float(gcrn_base['pesq_mix'])) + ' '
    #         + 'gcrn:' + "{:.2f}".format(float(gcrn_base['pesq_est'])) + ' '
    #         + 'dpcrn:' + "{:.2f}".format(float(dpcrn_base['pesq_est'])) + ' '
    #         + 'mixformer:' + "{:.2f}".format(float(mixformer_pro['pesq_est'])) + ' '
    #         + 'depthwisefirst:' + "{:.2f}".format(float(depthwisefirst_pro['pesq_est'])) + ' '
    #         + 'noattention:' + "{:.2f}".format(float(noattention_pro['pesq_est'])) + ' '
    #         + 'nodprnn:' + "{:.2f}".format(float(nodprnn_pro['pesq_est'])) + ' '
    #
    #     )
    #
    #     print(test_name + " STOI:")
    #     print(
    #         'mix:' + "{:.2f}".format(float(gcrn_base['stoi_mix']) * 100) + ' '
    #         + 'gcrn:' + "{:.2f}".format(float(gcrn_base['stoi_est']) * 100) + ' '
    #         + 'dpcrn:' + "{:.2f}".format(float(dpcrn_base['stoi_est']) * 100) + ' '
    #         + 'mixformer:' + "{:.2f}".format(float(mixformer_pro['stoi_est']) * 100) + ' '
    #         + 'depthwisefirst:' + "{:.2f}".format(float(depthwisefirst_pro['stoi_est']) * 100) + ' '
    #         + 'noattention:' + "{:.2f}".format(float(noattention_pro['stoi_est']) * 100) + ' '
    #         + 'nodprnn:' + "{:.2f}".format(float(nodprnn_pro['stoi_est']) * 100) + ' '
    #     )

    # 8、不同信噪比下的实验结果：
    # test_list = [
    #     'test_0.2_snr-10', 'test_0.2_snr-5', 'test_0.2_snr0', 'test_0.2_snr5', 'test_0.2_snr10',
    #     'test_0.3_snr-10', 'test_0.3_snr-5', 'test_0.3_snr0', 'test_0.3_snr5', 'test_0.3_snr10',
    #     'test_0.4_snr-10', 'test_0.4_snr-5', 'test_0.4_snr0', 'test_0.4_snr5', 'test_0.4_snr10',
    #     'test_0.5_snr-10', 'test_0.5_snr-5', 'test_0.5_snr0', 'test_0.5_snr5', 'test_0.5_snr10',
    #     'test_0.6_snr-10', 'test_0.6_snr-5', 'test_0.6_snr0', 'test_0.6_snr5', 'test_0.6_snr10',
    #     'test_0.7_snr-10', 'test_0.7_snr-5', 'test_0.7_snr0', 'test_0.7_snr5', 'test_0.7_snr10',
    #     'test_0.8_snr-10', 'test_0.8_snr-5', 'test_0.8_snr0', 'test_0.8_snr5', 'test_0.8_snr10',
    #     'test_0.9_snr-10', 'test_0.9_snr-5', 'test_0.9_snr0', 'test_0.9_snr5', 'test_0.9_snr10',
    #     'test_1.0_snr-10', 'test_1.0_snr-5', 'test_1.0_snr0', 'test_1.0_snr5', 'test_1.0_snr10',
    # ]
    # snr_list = [-10, -5, 0, 5, 10]
    # t60_list = [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
    # for snr in snr_list:
    #     pesq_mix_avg, pesq_gcrn_avg, pesq_dpcrn_avg, pesq_pro_avg, pesq_no_avg = 0, 0, 0, 0, 0
    #     stoi_mix_avg, stoi_gcrn_avg, stoi_dpcrn_avg, stoi_pro_avg, stoi_no_avg = 0, 0, 0, 0, 0
    #
    #     for t60 in t60_list:
    #         test_name = 'test_' + str(t60) + '_snr' + str(snr)
    #         gcrnpath = '/home/imu_panjiahui/MixFormer/GCRN_baseline/model_gcrn_baseline/result/' + test_name
    #         dpcrnpath = '/home/imu_panjiahui/MixFormer/DPCRN_baseline/model_dpcrn/result/' + test_name
    #         depthwisefirstpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer/model_dpcrn_mixformer_noattention_depthfirst/result/' + test_name
    #         noInteractionspath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer_noInteractions/model_dpcrn_mixformer_noattention_depthfirst_noInteractions/result/' + test_name
    #
    #         gcrn_base = scipy.io.loadmat(gcrnpath + '_metrics.mat')
    #         dpcrn_base = scipy.io.loadmat(dpcrnpath + '_metrics.mat')
    #         depthwisefirst_pro = scipy.io.loadmat(depthwisefirstpath + '_metrics.mat')
    #         noInteractions = scipy.io.loadmat(noInteractionspath + '_metrics.mat')
    #
    #         pesq_mix_avg += gcrn_base['pesq_mix']
    #         pesq_gcrn_avg += gcrn_base['pesq_est']
    #         pesq_dpcrn_avg += dpcrn_base['pesq_est']
    #         pesq_pro_avg += depthwisefirst_pro['pesq_est']
    #         pesq_no_avg += noInteractions['pesq_est']
    #
    #         stoi_mix_avg += gcrn_base['stoi_mix']
    #         stoi_gcrn_avg += gcrn_base['stoi_est']
    #         stoi_dpcrn_avg += dpcrn_base['stoi_est']
    #         stoi_pro_avg += depthwisefirst_pro['stoi_est']
    #         stoi_no_avg += noInteractions['stoi_est']
    #
    #     print('snr is: ' + str(snr))
    #     print(
    #         'pesq: ' +
    #         'mix:' + "{:.2f}".format(float(pesq_mix_avg / len(t60_list))) + ' ' +
    #         'gcrn:' + "{:.2f}".format(float(pesq_gcrn_avg / len(t60_list))) + ' ' +
    #         'dpcrn:' + "{:.2f}".format(float(pesq_dpcrn_avg / len(t60_list))) + ' ' +
    #         'pro:' + "{:.2f}".format(float(pesq_pro_avg / len(t60_list))) + ' ' +
    #         'no:' + "{:.2f}".format(float(pesq_no_avg / len(t60_list))) + ' ' +
    #         'stoi: ' +
    #         'mix:' + "{:.2f}".format(float(stoi_mix_avg / len(t60_list)) * 100) + ' ' +
    #         'gcrn:' + "{:.2f}".format(float(stoi_gcrn_avg / len(t60_list)) * 100) + ' ' +
    #         'dpcrn:' + "{:.2f}".format(float(stoi_dpcrn_avg / len(t60_list)) * 100) + ' ' +
    #         'pro:' + "{:.2f}".format(float(stoi_pro_avg / len(t60_list)) * 100) + ' ' +
    #         'no:' + "{:.2f}".format(float(stoi_no_avg / len(t60_list)) * 100) + ' '
    #
    #     )

    # 不同信噪比下，T60的变化情况
    snr_list = [-10, -5, 0, 5, 10]
    t60_list = [0.2]
    for snr in snr_list:
        print('SNR: ' + str(snr))
        for t60 in t60_list:
            test_name = 'test_' + str(t60) + '_snr' + str(snr)
            gcrnpath = '/home/imu_panjiahui/MixFormer/GCRN_baseline/model_gcrn_baseline/result/' + test_name
            dpcrnpath = '/home/imu_panjiahui/MixFormer/DPCRN_baseline/model_dpcrn/result/' + test_name
            depthwisefirstpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer/model_dpcrn_mixformer_noattention_depthfirst/result/' + test_name

            noInteractionspath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer_noInteractions/model_dpcrn_mixformer_noattention_depthfirst_noInteractions/result/' + test_name
            noParllelpath = '/home/imu_panjiahui/MixFormer/DPCRN_mixformer_noParallel/model_dpcrn_mixformer_noattention_depthfirst_noParallel/result/' + test_name

            gcrn_base = scipy.io.loadmat(gcrnpath + '_metrics.mat')
            dpcrn_base = scipy.io.loadmat(dpcrnpath + '_metrics.mat')
            depthwisefirst_pro = scipy.io.loadmat(depthwisefirstpath + '_metrics.mat')
            noInteractions = scipy.io.loadmat(noInteractionspath + '_metrics.mat')
            noParllel = scipy.io.loadmat(noParllelpath + '_metrics.mat')

            print(
                'PESQ: '
                + 't60: ' + str(t60) + ' '
                # + 'mix:' + "{:.2f}".format(float(gcrn_base['pesq_mix'])) + ' '
                # + 'gcrn:' + "{:.2f}".format(float(gcrn_base['pesq_est'])) + ' '
                + 'dpcrn:' + "{:.2f}".format(float(dpcrn_base['pesq_est'])) + ' '
                + 'noInteractions:' + "{:.2f}".format(float(noInteractions['pesq_est'])) + ' '
                + 'depthwisefirst:' + "{:.2f}".format(float(depthwisefirst_pro['pesq_est'])) + ' '

                # + 'noParllel:' + "{:.2f}".format(float(noParllel['pesq_est'])) + ' '
            )
            print(
                'STOI: '
                + 't60: ' + str(t60) + ' '
                # + 'mix:' + "{:.2f}".format(float(gcrn_base['stoi_mix']) * 100) + ' '
                # + 'gcrn:' + "{:.2f}".format(float(gcrn_base['stoi_est']) * 100) + ' '
                + 'dpcrn:' + "{:.2f}".format(float(dpcrn_base['stoi_est']) * 100) + ' '
                + 'noInteractions:' + "{:.2f}".format(float(noInteractions['stoi_est']) * 100) + ' '
                + 'depthwisefirst:' + "{:.2f}".format(float(depthwisefirst_pro['stoi_est']) * 100) + ' '

                # + 'noParllel:' + "{:.2f}".format(float(noParllel['stoi_est']) * 100) + ' '
            )