# -*- coding: utf-8 -*-
from collections import OrderedDict
from io import BytesIO

import numpy as np
import pytest

import megengine as mge
import megengine.functional as F
from megengine import Parameter, Tensor, tensor
from megengine.device import get_device_count
from megengine.module import (
    BatchNorm1d,
    BatchNorm2d,
    Conv1d,
    Conv2d,
    Dropout,
    GroupNorm,
    InstanceNorm,
    Linear,
    MaxPool2d,
    Module,
    Sequential,
    Softmax,
)
from megengine.module.module import _access_structure
from megengine.quantization.quantize import quantize, quantize_qat
from megengine.traced_module import TracedModule, trace_module
from megengine.utils.module_utils import get_expand_structure, set_expand_structure

class MLP(Module):
    def __init__(self):
        super().__init__()
        self.dense0 = Linear(28, 50)
        self.dense1 = Linear(50, 20)

    def forward(self, x):
        x = self.dense0(x)
        x = F.relu(x)
        x = self.dense1(x)
        return x

class MyModule(Module):
    class InnerModule(Module):
        def __init__(self):
            super().__init__()
            self.bn = BatchNorm2d(4)

        def forward(self, x):
            return self.bn(x)

    def __init__(self):
        super().__init__()
        self.i = self.InnerModule()
        self.bn = BatchNorm2d(4)
        self.param = Parameter(np.ones(1, dtype=np.float32))
        self.buff = Tensor(np.ones(1, dtype=np.float32))

    def forward(self, x):
        x = self.i(x)
        x = self.bn(x)
        return x

@pytest.mark.parametrize("test_traced_module", [True, False])
def test_module_api(test_traced_module):
    m = MyModule()
    if test_traced_module:
        buff = m.buff
        param = m.param
        m = trace_module(m, Tensor(np.random.random((1, 4, 16, 16))))
        assert "buff" not in m.__dict__
        assert "param" not in m.__dict__
        m.buff = buff
        m.param = param

    assert list(m.children()) == [m.bn, m.i]
    assert list(m.named_children()) == [("bn", m.bn), ("i", m.i)]
    assert list(m.modules()) == [m, m.bn, m.i, m.i.bn]
    assert list(m.named_modules()) == [
        ("", m),
        ("bn", m.bn),
        ("i", m.i),
        ("i.bn", m.i.bn),
    ]
    assert list(m.named_modules(prefix="x")) == [
        ("x", m),
        ("x.bn", m.bn),
        ("x.i", m.i),
        ("x.i.bn", m.i.bn),
    ]
    assert list(m.buffers()) == [
        m.bn.running_mean,
        m.bn.running_var,
        m.buff,
        m.i.bn.running_mean,
        m.i.bn.running_var,
    ]
    assert list(m.buffers(recursive=False)) == [m.buff]
    assert list(m.named_buffers()) == [
        ("bn.running_mean", m.bn.running_mean),
        ("bn.running_var", m.bn.running_var),
        ("buff", m.buff),
        ("i.bn.running_mean", m.i.bn.running_mean),
        ("i.bn.running_var", m.i.bn.running_var),
    ]
    assert list(m.parameters()) == [
        m.bn.bias,
        m.bn.weight,
        m.i.bn.bias,
        m.i.bn.weight,
        m.param,
    ]
    assert list(m.named_parameters()) == [
        ("bn.bias", m.bn.bias),
        ("bn.weight", m.bn.weight),
        ("i.bn.bias", m.i.bn.bias),
        ("i.bn.weight", m.i.bn.weight),
        ("param", m.param),
    ]
    assert list(m.tensors()) == [
        m.bn.bias,
        m.bn.running_mean,
        m.bn.running_var,
        m.bn.weight,
        m.buff,
        m.i.bn.bias,
        m.i.bn.running_mean,
        m.i.bn.running_var,
        m.i.bn.weight,
        m.param,
    ]
    assert list(m.named_tensors()) == [
        ("bn.bias", m.bn.bias),
        ("bn.running_mean", m.bn.running_mean),
        ("bn.running_var", m.bn.running_var),
        ("bn.weight", m.bn.weight),
        ("buff", m.buff),
        ("i.bn.bias", m.i.bn.bias),
        ("i.bn.running_mean", m.i.bn.running_mean),
        ("i.bn.running_var", m.i.bn.running_var),
        ("i.bn.weight", m.i.bn.weight),
        ("param", m.param),
    ]
    m.eval()
    assert (
        m.training == False
        and m.bn.training == False
        and m.i.training == False
        and m.i.bn.training == False
    )
    m.bn.train()
    assert m.training == False and m.bn.training == True and m.i.bn.training == False
    m.eval()
    m.i.train()
    assert (
        m.training == False
        and m.bn.training == False
        and m.i.training == True
        and m.i.bn.training == True
    )
    m.eval()
    m.train()
    assert m.training == True and m.bn.training == True and m.i.bn.training == True

    def fn(m):
        m.training = False

    m.apply(fn)
    assert m.bn.training == False and m.i.bn.training == False

@pytest.mark.parametrize("test_traced_module", [True, False])
def test_module_api_reuse_submodule(test_traced_module):
    m = MyModule()
    if test_traced_module:
        m = trace_module(m, Tensor(np.random.random((1, 4, 16, 16))))
    m.h = m.i  # pylint: disable=attribute-defined-outside-init
    assert list(m.modules()) == [m, m.bn, m.i, m.i.bn]
    assert list(m.named_modules()) == [
        ("", m),
        ("bn", m.bn),
        ("h", m.i),
        ("h.bn", m.i.bn),
    ]

@pytest.mark.parametrize("test_traced_module", [True, False])
def test_module_api_iterable_stability(test_traced_module):
    m = MyModule()
    if test_traced_module:
        m = trace_module(m, Tensor(np.random.random((1, 4, 16, 16))))
    l = list(m.modules())
    for _ in range(100):
        assert list(m.modules()) == l

@pytest.mark.parametrize("test_traced_module", [True, False])
def test_module_api_hooks(test_traced_module):
    net = MyModule()
    if test_traced_module:
        net = trace_module(net, Tensor(np.zeros((1, 4, 1, 1))))
    pre_hook_num = 0
    post_hook_num = 0
    hooks = []

    def pre_hook(_, inputs):
        nonlocal pre_hook_num
        pre_hook_num += 1
        modified_inputs = tuple(inp + 1 for inp in inputs)
        return modified_inputs

    def post_hook(_, __, outputs):
        nonlocal post_hook_num
        post_hook_num += 1
        outputs += 1
        return outputs

    net.apply(lambda module: hooks.append(module.register_forward_pre_hook(pre_hook)))
    net.apply(lambda module: hooks.append(module.register_forward_hook(post_hook)))

    shape = (1, 4, 1, 1)
    x = tensor(np.zeros(shape, dtype=np.float32))
    y = net(x)

    assert pre_hook_num == 4
    assert post_hook_num == 4
    mean1 = Parameter(np.zeros(shape), dtype=np.float32)
    bn1 = F.batch_norm(
        x + 3, mean1, Parameter(np.ones(shape), dtype=np.float32), training=True
    )
    np.testing.assert_allclose(
        net.i.bn.running_mean.numpy(), mean1.numpy(),
    )
    mean2 = Parameter(np.zeros(shape), dtype=np.float32)
    bn2 = F.batch_norm(
        bn1 + 3, mean2, Parameter(np.ones(shape), dtype=np.float32), training=True
    )
    np.testing.assert_allclose(
        net.bn.running_mean.numpy(), mean2.numpy(),
    )
    np.testing.assert_allclose((bn2 + 2).numpy(), y.numpy())

    assert len(hooks) == 8
    for handler in hooks:
        handler.remove()
    y = net(x)
    assert pre_hook_num == 4
    assert post_hook_num == 4

class MyModule2(Module):
    class InnerModule(Module):
        def __init__(self):
            super().__init__()
            self.bn = BatchNorm2d(4)
            self.test_bool_key = {True: 1, False: 0}

        def forward(self, x):
            x = self.bn(x)

    def __init__(self):
        super().__init__()
        self.bn = BatchNorm2d(4)
        self.a = [
            BatchNorm2d(4),
            {"x": BatchNorm2d(4), "y": [BatchNorm2d(4), self.InnerModule()], "z": 0},
            (self.InnerModule(),),
        ]

    def forward(self, x):
        return x

def test_expand_structure():
    m = MyModule2()
    rst = [
        ("", m),
        ("a.0", m.a[0]),
        ("a.1.x", m.a[1]["x"]),
        ("a.1.y.0", m.a[1]["y"][0]),
        ("a.1.y.1", m.a[1]["y"][1]),
        ("a.1.y.1.bn", m.a[1]["y"][1].bn),
        ("a.2.0", m.a[2][0]),
        ("a.2.0.bn", m.a[2][0].bn),
        ("bn", m.bn),
    ]
    assert list(m.named_modules()) == rst

    for item in rst[1:]:
        assert get_expand_structure(m, item[0]) == item[1]

    for item in reversed(rst[1:]):
        if _access_structure(m, item[0], lambda p, k, o: isinstance(p, tuple)):
            continue
        set_expand_structure(m, item[0], "TEST_VALUE")
        assert get_expand_structure(m, item[0]) == "TEST_VALUE"

def test_flatten_others():
    def be_others(obj):
        return not isinstance(obj, (Tensor, Module))

    m = MyModule2()
    assert len(list(m._flatten(with_key=True, predicate=be_others))) == 0

def test_flatten_with_parent():
    m = MyModule2()
    assert list(m.named_modules(with_parent=True)) == [
        ("", m, None),
        ("a.0", m.a[0], m),
        ("a.1.x", m.a[1]["x"], m),
        ("a.1.y.0", m.a[1]["y"][0], m),
        ("a.1.y.1", m.a[1]["y"][1], m),
        ("a.1.y.1.bn", m.a[1]["y"][1].bn, m.a[1]["y"][1]),
        ("a.2.0", m.a[2][0], m),
        ("a.2.0.bn", m.a[2][0].bn, m.a[2][0]),
        ("bn", m.bn, m),
    ]
    assert list(m.modules(with_parent=True)) == [
        (m, None),
        (m.a[0], m),
        (m.a[1]["x"], m),
        (m.a[1]["y"][0], m),
        (m.a[1]["y"][1], m),
        (m.a[1]["y"][1].bn, m.a[1]["y"][1]),
        (m.a[2][0], m),
        (m.a[2][0].bn, m.a[2][0]),
        (m.bn, m),
    ]

class MyModule3(Module):
    class InnerModule(Module):
        def __init__(self):
            super().__init__()
            self.bn = BatchNorm2d(4)

        def forward(self, x):
            x = self.bn(x)

    def __init__(self):
        super().__init__()
        self.bn = BatchNorm2d(4)
        self.seq = Sequential(BatchNorm2d(4), self.InnerModule(),)

    def forward(self, x):
        return x

def test_module_api_with_sequential():
    m = MyModule3()
    assert list(m.named_modules()) == [
        ("", m),
        ("bn", m.bn),
        ("seq", m.seq),
        ("seq.0", m.seq[0]),
        ("seq.1", m.seq[1]),
        ("seq.1.bn", m.seq[1].bn),
    ]

def test_sequential_named_children():
    modules = OrderedDict()
    modules["name0"] = Linear(20, 10)
    modules["name1"] = Linear(10, 5)
    modules["name2"] = Linear(5, 1)
    m = Sequential(modules)
    l = list(m.named_children())
    assert l[0][0] == "name0"
    assert l[1][0] == "name1"
    assert l[2][0] == "name2"

def test_state_dict():
    data_shape = (2, 28)
    data = tensor(np.random.random(data_shape))
    mlp = MLP()
    pred0 = mlp(data)

    with BytesIO() as fout:
        mge.save(mlp.state_dict(), fout)
        fout.seek(0)
        state_dict = mge.load(fout)
        state_dict["extra"] = None
        mlp1 = MLP()
        mlp1.load_state_dict(state_dict, strict=False)
        pred1 = mlp1(data)
        np.testing.assert_allclose(pred0.numpy(), pred1.numpy(), atol=5e-6)
        with pytest.raises(KeyError):
            mlp1.load_state_dict(state_dict)
        del state_dict["extra"]
        del state_dict["dense0.bias"]
        with pytest.raises(KeyError):
            mlp1.load_state_dict(state_dict)

class AssertModule(Module):
    def __init__(self):
        super().__init__()
        self.error_tensor_key = {True: tensor([]), False: 0}

    def forward(self, x):
        return x

def test_assert_message():
    with pytest.raises(
        AssertionError, match="keys for Tensor and Module must be str, error key: True"
    ):
        m = AssertModule()
        list(m._flatten())

class Simple(Module):
    def __init__(self):
        super().__init__()
        self.conv0 = Conv2d(1, 1, kernel_size=3, bias=False)
        self.conv1 = Conv2d(1, 1, kernel_size=3, bias=False)
        self.conv1.weight = self.conv0.weight

    def forward(self, inputs):
        x = self.conv0(inputs)
        y = self.conv1(inputs)
        return x + y

@pytest.mark.parametrize("test_traced_module", [True, False])
def test_shared_param(test_traced_module):
    net = Simple()
    if test_traced_module:
        net = trace_module(net, tensor(np.random.random((1, 1, 8, 8))))
    assert net.conv0.weight is net.conv1.weight
    data = tensor(np.random.random((1, 1, 8, 8)).astype(np.float32))
    np.testing.assert_allclose(net.conv0(data).numpy(), net.conv1(data).numpy())
    with BytesIO() as f:
        mge.save(net, f)
        f.seek(0)
        net1 = mge.load(f)
    assert net1.conv0.weight is net1.conv1.weight
    np.testing.assert_allclose(net1.conv0(data).numpy(), net1.conv1(data).numpy())

    with BytesIO() as f:
        mge.save(net.conv0, f)
        f.seek(0)
        conv0 = mge.load(f)

    with BytesIO() as f:
        mge.save(net.conv1, f)
        f.seek(0)
        conv1 = mge.load(f)

    assert conv0.weight is not conv1.weight
    np.testing.assert_allclose(conv0(data).numpy(), conv1(data).numpy())

class Simple2(Module):
    def __init__(self):
        super().__init__()
        self.conv1 = Conv1d(1, 1, kernel_size=3, bias=False)
        self.conv0 = Conv1d(1, 1, kernel_size=3, bias=False)
        self.conv1.weight = self.conv0.weight

    def forward(self, inputs):
        pass

def test_shared_param_1d():
    net = Simple2()
    assert net.conv0.weight is net.conv1.weight
    data = tensor(np.random.random((1, 1, 8)).astype(np.float32))
    np.testing.assert_allclose(net.conv0(data).numpy(), net.conv1(data).numpy())
    with BytesIO() as f:
        mge.save(net, f)
        f.seek(0)
        net1 = mge.load(f)
    assert net1.conv0.weight is net1.conv1.weight
    np.testing.assert_allclose(net1.conv0(data).numpy(), net1.conv1(data).numpy())

    with BytesIO() as f:
        mge.save(net.conv0, f)
        f.seek(0)
        conv0 = mge.load(f)

    with BytesIO() as f:
        mge.save(net.conv1, f)
        f.seek(0)
        conv1 = mge.load(f)

    assert conv0.weight is not conv1.weight
    np.testing.assert_allclose(conv0(data).numpy(), conv1(data).numpy())

@pytest.mark.parametrize("test_traced_module", [True, False])
def test_pickle_module(test_traced_module):
    data_shape = (2, 28)
    data = tensor(np.random.random(data_shape))
    mlp = MLP()
    pred_gt = mlp(data)
    if test_traced_module:
        mlp = trace_module(mlp, data)
    # pickle before forward
    with BytesIO() as fout:
        mge.save(mlp, fout)
        fout.seek(0)
        mlp1 = mge.load(fout)
        if test_traced_module:
            assert type(mlp1) == TracedModule
        pred0 = mlp1(data)

    pred1 = mlp(data)

    # pickle after forward
    with BytesIO() as fout:
        mge.save(mlp, fout)
        fout.seek(0)
        mlp1 = mge.load(fout)
        if test_traced_module:
            assert type(mlp1) == TracedModule
        pred2 = mlp1(data)

    np.testing.assert_allclose(pred_gt.numpy(), pred1.numpy(), atol=5e-6)
    np.testing.assert_allclose(pred0.numpy(), pred1.numpy(), atol=5e-6)
    np.testing.assert_allclose(pred0.numpy(), pred2.numpy(), atol=5e-6)

def test_repr_basic():
    # test whether __repr__ can output correct information
    class ConvModel(Module):
        def __init__(self):
            super().__init__()
            self.conv1 = Conv2d(3, 128, 3, padding=1, bias=False)
            self.conv2 = Conv2d(3, 128, 3, dilation=2, bias=False)
            self.bn1 = BatchNorm1d(128)
            self.bn2 = BatchNorm2d(128)
            self.pooling = MaxPool2d(kernel_size=2, padding=0)
            modules = OrderedDict()
            modules["depthwise"] = Conv2d(256, 256, 3, 1, 0, groups=256, bias=False,)
            modules["pointwise"] = Conv2d(
                256, 256, kernel_size=1, stride=1, padding=0, bias=True,
            )
            self.submodule1 = Sequential(modules)
            self.list1 = [Dropout(drop_prob=0.1), [Softmax(axis=100)]]
            self.tuple1 = (
                Dropout(drop_prob=0.1),
                (Softmax(axis=100), Dropout(drop_prob=0.2)),
            )
            self.dict1 = {"Dropout": Dropout(drop_prob=0.1)}
            self.fc1 = Linear(512, 1024)

        def forward(self, inputs):
            pass

    ground_truth = (
        "ConvModel(\n"
        "  (conv1): Conv2d(3, 128, kernel_size=(3, 3), padding=(1, 1), bias=False)\n"
        "  (conv2): Conv2d(3, 128, kernel_size=(3, 3), dilation=(2, 2), bias=False)\n"
        "  (bn1): BatchNorm1d(128, eps=1e-05, momentum=0.9, affine=True, track_running_stats=True)\n"
        "  (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.9, affine=True, track_running_stats=True)\n"
        "  (pooling): MaxPool2d(kernel_size=2, stride=2, padding=0)\n"
        "  (submodule1): Sequential(\n"
        "    (depthwise): Conv2d(256, 256, kernel_size=(3, 3), groups=256, bias=False)\n"
        "    (pointwise): Conv2d(256, 256, kernel_size=(1, 1))\n"
        "  )\n"
        "  (list1.0): Dropout(drop_prob=0.1)\n"
        "  (list1.1.0): Softmax(axis=100)\n"
        "  (tuple1.0): Dropout(drop_prob=0.1)\n"
        "  (tuple1.1.0): Softmax(axis=100)\n"
        "  (tuple1.1.1): Dropout(drop_prob=0.2)\n"
        "  (dict1.Dropout): Dropout(drop_prob=0.1)\n"
        "  (fc1): Linear(in_features=512, out_features=1024, bias=True)\n"
        ")"
    )
    net = ConvModel()
    output = net.__repr__()
    assert output == ground_truth

def test_repr_module_reassign():
    # test whether __repr__ can deal with module reassign
    class ConvModel1(Module):
        def __init__(self):
            super().__init__()
            self.conv1 = Conv2d(3, 128, 3, bias=False)
            self.conv2 = Conv2d(3, 128, 3, padding=1, bias=False)
            self.conv1 = Conv2d(3, 256, 3, dilation=2, bias=False)

        def forward(self, inputs):
            pass

    ground_truth = (
        "ConvModel1(\n"
        "  (conv1): Conv2d(3, 256, kernel_size=(3, 3), dilation=(2, 2), bias=False)\n"
        "  (conv2): Conv2d(3, 128, kernel_size=(3, 3), padding=(1, 1), bias=False)\n"
        ")"
    )
    net = ConvModel1()
    output = net.__repr__()
    assert output == ground_truth

def test_repr_module_rereference():
    # test whether __repr__ can deal with module re-reference
    class ConvModel2(Module):
        def __init__(self):
            super().__init__()
            self.conv1 = Conv2d(3, 128, 3, bias=False)
            self.conv2 = self.conv1
            self.conv3 = self.conv1

        def forward(self, inputs):
            pass

    ground_truth = (
        "ConvModel2(\n"
        "  (conv1): Conv2d(3, 128, kernel_size=(3, 3), bias=False)\n"
        "  (conv2): Conv2d(3, 128, kernel_size=(3, 3), bias=False)\n"
        "  (conv3): Conv2d(3, 128, kernel_size=(3, 3), bias=False)\n"
        ")"
    )
    net = ConvModel2()
    output = net.__repr__()
    assert output == ground_truth

def test_repr_module_delete():
    # test whether __repr__ can deal with module delete
    class ConvModel3(Module):
        def __init__(self):
            super().__init__()
            self.conv1 = Conv2d(3, 128, 3, bias=False)
            self.softmax = Softmax(100)

        def forward(self, inputs):
            pass

    ground_truth = (
        "ConvModel3(\n"
        "  (conv1): Conv2d(3, 128, kernel_size=(3, 3), bias=False)\n"
        ")"
    )
    net = ConvModel3()
    del net.softmax
    output = net.__repr__()
    assert output == ground_truth

def test_repr_module_reset_attr():
    class ResetAttrModule(Module):
        def __init__(self, flag):
            super().__init__()
            if flag:
                self.a = None
                self.a = Linear(3, 5)
            else:
                self.a = Linear(3, 5)
                self.a = None

        def forward(self, x):
            if self.a:
                x = self.a(x)
            return x

    ground_truth = [
        (
            "ResetAttrModule(\n"
            "  (a): Linear(in_features=3, out_features=5, bias=True)\n"
            ")"
        ),
        ("ResetAttrModule()"),
    ]

    m0 = ResetAttrModule(True)
    m1 = ResetAttrModule(False)
    output = [m0.__repr__(), m1.__repr__()]
    assert output == ground_truth

def test_module_compatible():
    class Empty(Module):
        def forward(self):
            pass

    empty_module = Empty()
    old_attributes = set(
        [
            "_modules",
            "name",
            "training",
            "quantize_disabled",
            "_forward_pre_hooks",
            "_forward_hooks",
            "_name",
            "_short_name",
        ]
    )
    current_attributes = set(empty_module.__dict__.keys())
    assert (
        old_attributes == current_attributes
    ), "Add or delete attributes in Module class may break compatibility of pickle serialization"

@pytest.mark.skip(reason="pytest aborted")
@pytest.mark.parametrize("affine", [True, False])
def test_grou_norm(affine):
    num_groups = 256
    num_channels = 256
    weight_np = np.random.uniform(-0.5, 0.5, (num_channels))
    bias_np = np.random.uniform(-0.5, 0.5, (num_channels))

    class OriginGroupNormFunc(Module):
        def __init__(self, eps=1e-5, affine=True, **kwargs):
            super().__init__(**kwargs)
            assert num_channels % num_groups == 0
            self.num_groups = num_groups
            self.num_channels = num_channels
            self.eps = eps
            self.affine = affine
            if self.affine:
                self.weight = Parameter(weight_np)
                self.bias = Parameter(bias_np)
            else:
                self.weight = None
                self.bias = None

        def forward(self, x):
            N, C, H, W = x.shape
            x = x.reshape(N, self.num_groups, -1)
            mean = x.mean(axis=2, keepdims=True)
            var = (x * x).mean(axis=2, keepdims=True) - mean * mean
            x = (x - mean) / F.sqrt(var + self.eps)
            x = x.reshape(N, C, H, W)
            if self.affine:
                x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(
                    1, -1, 1, 1
                )
            return x

    inp = np.random.uniform(-0.5, 0.5, (2, num_channels, 10, 16)).astype("float32")
    mge_inp = Tensor(inp)
    mge_m = GroupNorm(num_groups, num_channels, affine=affine)
    mge_m.weight = Parameter(weight_np)
    mge_m.bias = Parameter(bias_np)
    ori_inp = Tensor(inp)
    ori_m = OriginGroupNormFunc(affine=affine)

    mge_gm = mge.autodiff.GradManager().attach((*mge_m.parameters(), mge_inp))
    ori_gm = mge.autodiff.GradManager().attach((*ori_m.parameters(), ori_inp))
    dy = Tensor(np.random.uniform(-0.5, 0.5, inp.shape))
    for i in range(2):
        with mge_gm:
            mge_output = mge_m(mge_inp)

            mge_gm.backward(mge_output, dy)

        with ori_gm:
            ori_output = ori_m(ori_inp)

            ori_gm.backward(ori_output, dy)

        np.testing.assert_allclose(mge_output.numpy(), ori_output.numpy(), atol=1e-05)
        np.testing.assert_allclose(
            ori_inp.grad.numpy(), mge_inp.grad.numpy(), atol=1e-05
        )
        if affine == True:
            np.testing.assert_allclose(
                mge_m.weight.grad.numpy(), ori_m.weight.grad.numpy(), atol=1e-05
            )
            np.testing.assert_allclose(
                mge_m.bias.grad.numpy(), ori_m.bias.grad.numpy(), atol=1e-05
            )

@pytest.mark.parametrize("affine", [True, False])
def test_instance_norm(affine):
    num_channels = 4
    weight_np = np.random.uniform(-0.5, 0.5, (num_channels))
    bias_np = np.random.uniform(-0.5, 0.5, (num_channels))

    class OriginInstanceNormFunc(Module):
        def __init__(self, eps=1e-5, affine=True, **kwargs):
            super().__init__(**kwargs)
            self.num_channels = num_channels
            self.eps = eps
            self.affine = affine
            if self.affine:
                self.weight = Parameter(weight_np)
                self.bias = Parameter(bias_np)
            else:
                self.weight = None
                self.bias = None

        def forward(self, x):
            N, C, H, W = x.shape
            x = x.reshape(N, self.num_channels, -1)
            mean = x.mean(axis=2, keepdims=True)
            var = (x * x).mean(axis=2, keepdims=True) - mean * mean
            x = (x - mean) / F.sqrt(var + self.eps)
            x = x.reshape(N, C, H, W)
            if self.affine:
                x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(
                    1, -1, 1, 1
                )
            return x

    inp = np.random.uniform(-0.5, 0.5, (2, num_channels, 10, 16)).astype("float32")
    mge_inp = Tensor(inp)
    mge_m = InstanceNorm(num_channels, affine=affine)
    mge_m.weight = Parameter(weight_np)
    mge_m.bias = Parameter(bias_np)

    ori_inp = Tensor(inp)
    ori_m = OriginInstanceNormFunc(affine=affine)

    mge_im = mge.autodiff.GradManager().attach((*mge_m.parameters(), mge_inp))
    ori_im = mge.autodiff.GradManager().attach((*ori_m.parameters(), ori_inp))
    dy = Tensor(np.random.uniform(-0.5, 0.5, inp.shape))

    for i in range(2):
        with mge_im:
            mge_output = mge_m(mge_inp)

            mge_im.backward(mge_output, dy)

        with ori_im:
            ori_output = ori_m(ori_inp)

            ori_im.backward(ori_output, dy)

        np.testing.assert_allclose(mge_output.numpy(), ori_output.numpy(), atol=1e-05)
        np.testing.assert_allclose(
            ori_inp.grad.numpy(), mge_inp.grad.numpy(), atol=1e-04
        )
        if affine == True:
            np.testing.assert_allclose(
                mge_m.weight.grad.numpy(), ori_m.weight.grad.numpy(), atol=1e-04
            )
            np.testing.assert_allclose(
                mge_m.bias.grad.numpy(), ori_m.bias.grad.numpy(), atol=1e-04
            )


import os

#==============================================================================

class Node:
    def __init__(self,name):
        self.name=name
        self.label=name
        self.info=""
        self.edgeL=[]               # a collection of Edge items
        self.color=None
        self.colorOutline=None
        self.group=None

    @classmethod
    def Copy(cls,node):
        self=cls(node.name)
        self.label=node.label
        self.info=node.info
        self.color=node.color
        self.colorOutline=node.colorOutline
        self.group=node.group
        return self

class Edge:
    def __init__(self,target=None):
        self.target=target          # a Node item
        self.info=""
        self.label=''
        self.color=None
        self.style=None

    @classmethod
    def Copy(cls,edge):
        self=cls()
        self.info=edge.info
        self.label=edge.label
        self.color=edge.color
        self.style=edge.style
        return self

class NodeGroup:
    def __init__(self,name):
        self.name=name
        self.label=name
        self.group=None
        self.color=None
        self.edgeL=[]

#==============================================================================
def GetTopNodes(nodeL) :
    topNodeL=nodeL[:]
    for node in nodeL:
        for edge in node.edgeL :
            try:
                topNodeL.remove(edge.target)
            except ValueError :
                pass
    return topNodeL

def GetEndNodes(nodeL) :
    endNodeL=[]
    for node in nodeL:
        if len(node.edgeL)==0 :
            endNodeL.append(node)
    return endNodeL

#==============================================================================

def NodeL2File(nodeL,filename=None, format=None):
    if not filename : filename='output.gml'
    fileName=os.path.splitext(filename)[0]
    ext=os.path.splitext(filename)[1]
    if ext=='': ext ='.gml'

    format2ext={'gml':'.gml','graph':'.graphml','graphml':'.graphml','dot':'.dot','txt':'.txt'}
    if format :
        ext=format2ext[format]

    if ext=='.gml' :
        lines=NodeL2GML(nodeL)
    elif ext=='.graphml' :
        lines=NodeL2GraphML(nodeL)
    elif ext=='.dot' :
        lines=NodeL2DOT(nodeL)
    elif ext=='.txt' :
        lines=NodeL2txt(nodeL)
    else :
        lines=NodeL2GML(nodeL)

    WriteLines(lines,fileName+ext)

def WriteLines(lines,fileName) :

    if len(fileName)>255:
        ext=os.path.splitext(fileName)[1]
        name=os.path.splitext(fileName)[0]
        fileName=fileName[:254-len(ext)]+'~'+ext

    f=open(fileName,'w')
    f.write('\n'.join(lines))
    f.close()

def HighlightNode(nodeL, MatchFunc, color):
    for node in filter(MatchFunc, nodeL) :
        node.color=color

#==============================================================================

def GetGroups(nodeL):
    groupL=[node.group for node in nodeL if node.group]
    groupL=list(set(groupL))
    return groupL

#==============================================================================

def NodeL2GML(nodeL):
    def FormatString(string):
        out=string
        out=out.replace('&','&')
        out=out.replace('"','"')
        return out

    colorD={'light_gray':'#B0B0B0','gray':'#808080', "blue0": '#6666ff' , 'blue1':'#ccccff','orange':'#ff7700','dgreen':'#008000','red':'#ff0000'}

    lines=[]
    lines.append('graph [')
    group=True

    nodeL_i=dict(zip(nodeL,range(len(nodeL))))

    groupL=GetGroups(nodeL)
    groupL_i=dict(zip(groupL,range(len(nodeL),len(nodeL)+len(groupL))))

    for node in nodeL:
        lines.append('\t'*1+'node [')
        lines.append('\t'*2+'id '+str(nodeL_i[node]) )
        lines.append('\t'*2+'label "'+ FormatString(node.label) +'"')
        lines.append('\t'*2+'graphics [')
        lines.append('\t'*3+'w '+str(15+7*(len(FormatString(node.label))+2)))
        if node.color :
            if node.color in colorD.keys():
                lines.append('\t'*3+'fill "'+colorD[node.color]+'"')
            else:
                lines.append('\t'*3+'fill "'+node.color+'"')
        if node.colorOutline :
            if node.colorOutline in colorD.keys():
                lines.append('\t'*3+'outline "'+colorD[node.colorOutline]+'"')
            else:
                lines.append('\t'*3+'outline "'+node.colorOutline+'"')

        lines.append('\t'*2+']')
        if node.group:
            lines.append('\t'*2+'gid '+str(groupL_i[node.group]))
        lines.append('\t'*1+']')

    for group in groupL:
        lines.append('\t'*1+'node [')
        lines.append('\t'*2+'id '+str(groupL_i[group]) )
        lines.append('\t'*2+'label "'+ FormatString(group.label) +'"')
        if group.group:
            lines.append('\t'*1+'gid '+str(groupL_i[group.group]))
        lines.append('\t'*1+'isGroup 1')
        lines.append('\t'*1+']')

    for node in nodeL:
        for edge in node.edgeL :
            lines.append('\t'*1+'edge [')
            lines.append('\t'*2+'source '+ str(nodeL_i[node]) )
            lines.append('\t'*2+'target '+ str(nodeL_i[edge.target]))
            if edge.label!='' :
                lines.append('\t'*2+'label "'+ FormatString(edge.label)+'"')

            lines.append('\t'*2+'graphics [')
            lines.append('\t'*3+'arrow "last"')
            if edge.color:
                if edge.color in colorD.keys():
                    lines.append('\t'*3+'fill "'+colorD[edge.color]+'"')
                else :
                    lines.append('\t'*3+'fill "'+edge.color+'"')
            if edge.style :
                lines.append('\t'*3+'style "'+edge.style+'"')
            lines.append('\t'*2+']')
            lines.append('\t'*1+']')

    lines.append(']')

    return lines

def NodeL2GraphML(nodeL):
    colorD={'light_gray':'#B0B0B0','gray':'#808080', "blue0": '#6666ff' , 'blue1':'#ccccff','orange':'#ff7700','dgreen':'#008000'}

    lines=[]
    lines.append('<graphml>')
    lines.append('<key id="color" for="node" attr.name="color" attr.type="string"/>')
    lines.append('<key id="label" for="node" attr.name="label" attr.type="string"/>')
    lines.append('<key id="info" for="node" attr.name="info" attr.type="string"/>')
    lines.append('<key id="ecolor" for="edge" attr.name="color" attr.type="string"/>')
    lines.append('<key id="einfo" for="edge" attr.name="info" attr.type="string"/>')
    lines.append('\t'*1+'<graph>')

    nodeL_i=dict(zip(nodeL,range(len(nodeL))))
    for node in nodeL:
        lines.append('\t'*2+'<node id="n'+str(nodeL_i[node])+'" >')
        lines.append('\t'*3+'<data key="label">'+ node.label+'</data>')
        if node.color:
            lines.append('\t'*3+'<data key="color">'+colorD[node.color]+'</data>')
        if node.info!=''  :
            lines.append('\t'*3+'<data key="info">'+node.info+'</data>')
        lines.append('\t'*2+'</node>')

    for node in nodeL:
        for edge in node.edgeL :
            lines.append('\t'*2+'<edge source="n'+str(nodeL_i[node])+'" target="n'+str(nodeL_i[edge.target])+'">')
            if edge.color:
                lines.append('\t'*3+'<data key="ecolor">'+colorD[edge.color]+'</data>')
            if edge.info!=""  :
                lines.append('\t'*3+'<data key="einfo">'+edge.info+'</data>')
            lines.append('\t'*2+'</edge>')

    lines.append('\t'*1+'</graph>')
    lines.append('</graphml>')

    return lines

def NodeL2DOT(nodeL):

    lines=[]
    lines.append('digraph dotgraph {')

    for node in nodeL:
        for edge in node.edgeL :

            style=[]
            style_str=''
            if edge.label != '':
                style.append='label="'+edge.label+'" '
            if edge.color != '':
                style.append='color='+edge.color+' '
            if len(style)>0:
                style_str=' ['+','.join(style)+']'

            lines.append('\t'+node.label +' -> '+edge.target.label+ style_str+ ';')

    for node in nodeL:
        style=[]
        if node.color != '':
            style+='color="'+node.color

        if len(style)>0:
            lines.append('\t'+node.label+' ['+','.join(style)+']')

    lines.append('}')

    return lines

def NodeL2txt(nodeL):
    def FollowDown(node,lvl=0):
        lines=[]
        lines.append('\t'*lvl+node.name)
        for edge in node.edgeL:
            lines+=FollowDown(edge.target,lvl+1)
        return lines

    lines=[]

    topNodeL=GetTopNodes(nodeL)
    for topNode in topNodeL:
        lines+=FollowDown(topNode)

    return lines


import os
import csv
#create filepath for csv data
poll_csv = os.path.join(".", "election_data.csv")

#initialize variables for vote counts for each candidate & total
allvote = 0
khan = 0
li = 0
tool = 0
correy = 0

#open csv file

with open(poll_csv, newline = "") as csvfile:
    csvreader = csv.reader(csvfile, delimiter=',')
    csv_header = next(csvreader)
    #count votes
    for row in csvreader:
        #count total votes
        allvote += 1
        #count votes for each candidate by incrementing their counter variable for each row where the candidate cell matches
        if row[2] == "Khan":
            khan += 1
        elif row[2] == "Correy":
            correy += 1
        elif row[2] == "Li":
            li += 1
        elif row[2] == "O'Tooley":
            tool += 1

#calculate percentage for each candidate
per_k = round(khan/allvote, 5)
per_c = round(correy/allvote, 5)
per_l = round(li/allvote, 5)
per_t = round(tool/allvote, 5)

#determine winner
if khan > correy and khan > li and khan > tool:
    winner = "Khan"
elif correy > khan and correy > li and correy > tool:
    winner = "Correy"
elif li > khan and li > correy and li > tool:
    winner = "Li"
else:
    winner = "O'Tooley"

#print results to terminal
print("Election Results \n-------------------------")
print("Total Votes: " + str(allvote))
print("-------------------------")
print("Khan: " + "{:.3%}".format(per_k) + " (" + str(khan) + ")")
print("Correy: " "{:.3%}".format(per_c) + " (" + str(correy) + ")")
print("Li: " + "{:.3%}".format(per_l) + " (" + str(li) + ")")
print("O'Tooley: " + "{:.3%}".format(per_t) + " (" + str(tool) + ")")
print("-------------------------")
print("Winner: " + winner)

#print results to .txt file

with open("results.txt", "w") as result:
    result.write("Election Results \n-------------------------")
    result.write("\nTotal Votes: " + str(allvote))
    result.write("\n-------------------------")
    result.write("\nKhan: " + "{:.3%}".format(per_k) + " (" + str(khan) + ")")
    result.write("\nCorrey: " "{:.3%}".format(per_c) + " (" + str(correy) + ")")
    result.write("\nLi: " + "{:.3%}".format(per_l) + " (" + str(li) + ")")
    result.write("\nO'Tooley: " + "{:.3%}".format(per_t) + " (" + str(tool) + ")")
    result.write("\n-------------------------")
    result.write("\nWinner: " + winner)


import urllib.request, urllib.error, urllib.parse, traceback, re, random
urllib.request.socket.setdefaulttimeout(5.0)

DEBUG = 0

class Dorker(object):
    def __init__(self, engine='duck'):
        self._engine = engine
        self.search_engines = [] # available dorking search engines
        self.search_engines.append('duck')
        self.search_engines.append('startpage')
        self.search_engines.append('yahoo')
        self.search_engines.append('bing')
        self.agents = [] # user-agents
        try:
            f = open("core/fuzzing/user-agents.txt").readlines() # set path for user-agents
        except:
            f = open("fuzzing/user-agents.txt").readlines() # set path for user-agents when testing
        for line in f:
            self.agents.append(line)

    def dork(self, search):
        """
        Perform a search and return links.
        """
        if self._engine == 'bing': # works at 20-02-2011 -> 19-02-2016 -> 09-04-2018 -> 26-08-2019
            search_url = 'https://www.bing.com/search?q="' + str(search) + '"'
            print("\nSearching query:", urllib.parse.unquote(search_url))
        elif self._engine == 'yahoo': # works at 20-02-2011 -> 19-02-2016 -> -> 09-04-2018 -> 26-08-2019
            search_url = 'https://search.yahoo.com/search?q="' + str(search) + '"'
            print("\nSearching query:", urllib.parse.unquote(search_url))
        elif self._engine == 'duck': # works at 26-08-2019
            search_url = 'https://duckduckgo.com/html/'
            q = 'instreamset:(url):"' + str(search) + '"' # set query to search literally on results
            query_string = { 'q':q }
            print("\nSearching query:", urllib.parse.unquote(search_url) + " [POST: (" + q + ")]")
        elif self._engine == 'startpage': # works at 26-08-2019
            search_url = 'https://www.startpage.com/do/asearch'
            q = 'url:"' + str(search) + '"' # set query to search literally on results
            query_string = { 'cmd':'process_search', 'query':q }
            print("\nSearching query:", urllib.parse.unquote(search_url) + " [POST: (" + q + ")]")
        else:
            print("\n[Error] This search engine is not being supported!\n")
            print('-'*25)
            print("\n[Info] Use one from this list:\n")
            for e in self.search_engines:
                print("+ "+e)
            print("\n ex: xsser -d 'profile.asp?num=' --De 'duck'")
            print(" ex: xsser -l --De 'startpage'")
            print("\n[Info] Or try them all:\n\n ex: xsser -d 'news.php?id=' --Da\n")
        try:
            self.search_url = search_url
            user_agent = random.choice(self.agents).strip() # set random user-agent
            referer = '127.0.0.1' # set referer to localhost / WAF black magic!
            headers = {'User-Agent' : user_agent, 'Referer' : referer}
            if self._engine == 'bing' or self._engine == 'yahoo': # using GET
                req = urllib.request.Request(search_url, None, headers)
            elif self._engine == 'duck' or self._engine == 'startpage': # using POST
                data = urllib.parse.urlencode(query_string)
                req = urllib.request.Request(search_url, data, headers)
            html_data = urllib.request.urlopen(req).read().decode('utf8')
            print("\n[Info] Retrieving requested info...\n")
        except urllib.error.URLError as e:
            if DEBUG:
                traceback.print_exc()
            print("\n[Error] Cannot connect!")
            print("\n" + "-"*50)
            return
        if self._engine == 'bing':
            regex = '
<a href="(.+?)" h=' # regex magics 08/2019
        if self._engine == 'yahoo':
            regex = 'RU=(.+?)/RK=' # regex magics 08/2019
        if self._engine == 'duck':
            regex = '' # regex 08/2019
        if self._engine == 'startpage':
            regex = 'target="_blank">(.+?)' # regex magics 08/2019
        pattern = re.compile(regex)
        links = re.findall(pattern, html_data, flags=0)
        found_links = []
        if links:
            for link in links:
                link = urllib.parse.unquote(link)
                if self._engine == "yahoo":
                    if "RU=https://www.yahoo.com/" in link:
                        link = "" # invalid url
                if search.upper() in link.upper(): # parse that search query is on url
                    sep = search
                    link2 = link.split(sep,1)[0]
                    if link2 not in found_links: # parse that target is not duplicated
                        found_links.append(link)
        else:
            print("\n[Error] Not any link found for that query!")
        return found_links

if __name__ == '__main__':
    for a in ['bing', 'yahoo', 'duck', 'startpage']: # working at: 28/08/2019
        dork = Dorker(a)
        res = dork.dork("news.php?id=")
        if res:
            print("\n[+] Search Engine:", a, "| Found: ", len(res), "\n")
            for b in res:
                print(" *", b)


import torch
import glob
#import DAN module from model.py
from model import DAN
#import FastCountVectorizer
from fastCountVectorizer import FastCountVectorizer
#import IMDBDataset from datasets/imdbDataset.py
from datasets.imdbDataset import ImdbDataset

#import torch Dataset
from torch.utils.data import Dataset, DataLoader

import random

tokenizer = FastCountVectorizer('../data/fast_countvectorizer/top_million_ngrams.txt')

config = {
    'embed_dimension': 1000,
    'intermediate_dimension': 1000
}

model = DAN(
    num_embeddings=tokenizer.size()+1,
    embed_dimension=config['embed_dimension'],
    intermediate_dimension=config['intermediate_dimension']
)

#load the model from the checkpoint
old_state_dict = torch.load('model_best.pth.tar', map_location='cpu')['state_dict']
#rename all the keys in the state_dict to not contain "module."
new_state_dict = {}
for key in old_state_dict:
    new_state_dict[key.replace('module.', '')] = old_state_dict[key]
model.load_state_dict(new_state_dict)

#load the dataset
test_files = glob.glob("../data/imdb/test/*/*.txt")
#randomly sample 100 files from the test set
test_files = random.sample(test_files, 2000)
dataset = ImdbDataset(test_files, tokenizer)

#get the dataloader
dataloader = torch.utils.data.DataLoader(dataset, batch_size=4, shuffle=False)

model.eval()
num_correct, total = 0,0

#add zeros until a has length 1000
def pad_with_zeros(a):
    return a + [0] * (1000 - len(a))
def prepare_text(s):
    return torch.LongTensor([pad_with_zeros(tokenizer.tokenize(s))])

#iterate through each batch
for batch_idx, (data, target) in enumerate(dataloader):
    #load the batch
    #data, target = data.to(torch.device('cuda')), target.to(torch.device('cuda'))
    #make predictions
    output = model(data)
    #check if the prediction is correct
    _, predicted = torch.max(output.data, 1)

    _, target_value = torch.max(target, 1)
    #increment the number of correct predictions and the total number of predictions
    num_correct += (predicted == target_value).sum().item()
    total += target.size(0)

#print the accuracy
print("Accuracy: %.2f %%" % (100 * num_correct / total))


#__________________________________________________________TITLES_________________________________________________________#

import tkinter as tk
from tkinter import ttk, filedialog
from PIL import ImageTk, Image
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
import numpy as np
import main
import pandas
from tkinter import *
from tkinter.ttk import *
import matplotlib.ticker as ticker

root = tk.Tk()
root.title("GPA Calculator")
root.config(bg='#CACBD1')
#root.attributes('-fullscreen', True)
root.geometry('1280x800')
root.resizable(width=0, height=0)

#data to be stored for every .RUN file
index: int = 0
sec_data = None
grp_data = None
pages: list = None

# ROGER WILLIAMS LABEL
title_label = ttk.Label(root, text="   Roger Williams", font=("Times New Roman", 20), foreground="white", background="#1E3261")
title_label.grid(row=1, column=0, columnspan=5, pady=(10, 0), sticky="nsew")

# UNIVERSITY LABEL
univ_label = ttk.Label(root, text="  University", font=("Times New Roman", 27), foreground="#60ADF0", background="#1E3261")
univ_label.grid(row=2, column=0, columnspan=5, pady=(0, 45), sticky="nsew")

#__________________________________________________________TEXTBOXES______________________________________________________#

# TOP SPACER BOX
top_space = tk.Entry(root, width=500, bg="#1E3261", fg="#1E3261", bd=0)
top_space = top_space.grid(row=0, rowspan=2, columnspan=5, pady=(0,30) , column=0, sticky="nesw")

# FILE DIRECTORY BOX
dir_box = tk.Entry(root, width=80, font=("Arial", 14), bg="white", fg="#a3a3a3", bd=0)
dir_box.insert(0, " Enter file path")
dir_box.bind("<FocusIn>", lambda event: dir_box.delete(0, tk.END))
dir_box.grid(row=3, column=0, padx=20, pady=10, sticky="w")

# FILE CONTENTS BOX
file_box = tk.Text(root, height=6, font=("Arial", 14), bg="white", fg="#a3a3a3", bd=0)
file_box.insert(tk.END, ' File contents will be displayed here.\n\n If entering manually, use the following format (include quotations):\n\n "Last","First","Student ID","Grade"')
file_box.bind("<FocusIn>", lambda event: file_box.delete(1.0, tk.END))
file_box.grid(row=5, column=0, padx=20, pady=10, sticky="w", columnspan=2)

# CALCULATION BOX
calc_box = tk.Text(root, height=17, font=("Arial", 14), bg="white", fg="#a3a3a3", bd=0)
calc_box.bind("<FocusIn>", lambda event: calc_box.delete(1.0, tk.END))
calc_box.grid(row=6, column=0, padx=20, pady=10, sticky="nsew", columnspan=2)

# BOTTOM SPACER BOX
bot_space = tk.Entry(root, width=100, bg="#CACBD1", fg="#CACBD1", bd=0)
bot_space = bot_space.grid(row=7, column=0, padx=5, pady=5, sticky="ew")

# GRAPH BOX
graph_box = tk.Entry(root, width=100, bg="white", fg="white", bd=0)
graph_box.grid(row=3, rowspan=4, column=3, columnspan=2, padx=20, pady=10, sticky="nse")

#________________________________________________________FUNCTIONS________________________________________________________#

# Function to handle file selection from directory browser
def select_file():
    file_path = filedialog.askopenfilename()
    if file_path:
        dir_box.delete(0, tk.END)
        dir_box.insert(0, file_path)
        with open(file_path, "r") as file:
            file_contents = file.read()
            file_box.delete(1.0, tk.END)
            file_box.insert(tk.END, file_contents)

def generate_graph():
    global index, pages, grp_data, sec_data

    fig, ax = plt.subplots(figsize=(6, 4))

    grades = ['A', 'A-', 'B+', 'B', 'B-', 'C+', 'C', 'C-', 'D+', 'D', 'D-', 'F', 'I', 'P', 'NP', 'W']
    num_students = [0, 3, 5, 10, 12, 15, 20, 18, 10, 8, 4, 0, 2, 6, 4, 1]

    ax.bar(grades, num_students)

    # Add labels and title
    ax.grid(axis='y')
    ax.set_xlabel('Grades')
    ax.set_ylabel('Number of Students')
    ax.set_title('Distribution of Grades')
    ax.set_yticks(range(0, 51, 2))

    # Add bar chart to window
    canvas = FigureCanvasTkAgg(fig, master=graph_box)
    canvas.draw()
    canvas.get_tk_widget().pack(fill=tk.BOTH, expand=True)

def clear_text():
    dir_box.delete(0, tk.END)
    dir_box.insert(0, " Enter file path")
    file_box.delete(1.0, tk.END)
    file_box.insert(tk.END, ' File contents will be displayed here.\n\n If entering manually, use the following format (include quotations):\n\n "Last","First","Student ID","Grade"')
    calc_box.delete(1.0, tk.END)
    calc_box.insert(tk.END, "")

def clear_canvas():
    global graph_canvas, graph_fig, graph_ax
    # Clear the old graph if it exists
    if graph_fig is not None:
        graph_ax.clear()
        graph_fig = None
        graph_canvas.draw()
        graph_canvas.get_tk_widget().destroy()

# Function to exit program
def exit_program():
    root.destroy()

#__________________________________________________________BUTTONS________________________________________________________#

# LEFT BUTTON
left_button = tk.Button(root, width = 10, text="<<", bd=0, fg='white', bg='#1E3261')
left_button.grid(row = 3, columnspan=2, column = 3, pady=10, padx=250, sticky='w')

# BROWSE BUTTON
browse_button = ttk.Button(root, text="Browse", width=10)
browse_button.grid(row=4, column=0, pady=10, padx=20, sticky="w")

# CALCULATE BUTTON
calc_button = ttk.Button(root, text="Calculate", width=10)
calc_button.grid(row=4, column=0, pady=10, padx=20, sticky="n")

# CLEAR BUTTON
clear_button = ttk.Button(root, text="Clear", width=10)
clear_button.grid(row=4, column=0, pady=10, padx=20, sticky="e")

# RIGHT BUTTON
right_button = tk.Button(root, text=">>", width = 10, bd=0, fg='white', bg='#1E3261')
right_button.grid(row = 3, columnspan=2, column = 4, pady=10, padx=210, sticky='e')

# EXIT BUTTON
exit_button = ttk.Button(root, text="Exit", command=exit_program, width=10)
exit_button.grid(row=1, column=4, sticky="ne", padx=7, pady=7)

#___________________________________________________________FUNCTIONS________________________________________________________#

def displayData():
    global index, pages, grp_data, sec_data

    if pages == None:
        return

    generate_graph()
    datastr = ""
    if pages[index].endswith("GRP"):
        data2 = grp_data[pages[index]]

        datastr += "\n" + "Group:\t" + str(pages[index])
        datastr += "\n" + "Sections and Z-test:"

        ztestdata = sorted(data2["ztests"].items(), key=lambda x:x[1])
        for item in ztestdata:
            datastr += "\n\t" + str(item[0]) + ":\t " + str(item[1])
            if item[1] >= 2.0:
                datastr += "SIGNIFICANTLY HIGH"
            elif item[1] <= -2.0:
                datastr += "SIGNIFICANTLY LOW"
        datastr += "\n"

        datastr += "\n" + "Mean:\t" + str(data2["mean"])
        datastr += "\n" + "Standard Deviation:\t" + str(data2["stddev"])
        datastr += "\n" + "Number of Students:\t" + str(data2["numstudents"])
        datastr += "\n" + "Grade Counts:"

        gradeCounts = list(data2["gradecounts"].keys())
        gradeCounts.sort()
        for item in gradeCounts:
            datastr += "\n\t" + str(item) + ":\t" + str(data2["gradecounts"][item])

    elif pages[index].endswith("SEC"):
        data2 = sec_data[pages[index]]

        datastr += "\n" + "Section:\t" + str(pages[index])
        datastr += "\n" + "Credit Hours:\t" + str(data2["creditHours"]) + "\n"
        datastr += "\n" + "Mean:\t" + str(data2["mean"])
        datastr += "\n" + "Standard Deviation:\t" + str(data2["stddev"])
        datastr += "\n" + "Number of Students:\t" + str(data2["numstudents"])
        datastr += "\n" + "Grade Counts:"
        gradeCounts = list(data2["gradecounts"].keys())
        gradeCounts.sort()
        for item in gradeCounts:
            datastr += "\n\t" + str(item) + ":\t" + str(data2["gradecounts"][item])
        datastr += "\n" + "Students:"

        students: pandas.DataFrame = data2["data"]
        datastr += "\n\t%15s %15s %7s %6s %8s" % (
            students.columns[0], students.columns[1], students.columns[2], students.columns[3], students.columns[4])
        for i in range(len(students.index)):
            datastr += "\n\t%s %s %s %s %s" % (
                str(students.iloc[i][0]).strip().ljust(20, " "),
                str(students.iloc[i][1]).strip().ljust(20, " "),
                str(students.iloc[i][2]).strip().ljust(7, " "),
                str(students.iloc[i][3]).strip().ljust(6, " "),
                str(students.iloc[i][4]).strip().ljust(8, " ")
            )

        #datastr += data2["data"].to_string(col_space = 15)
    calc_box.delete("1.0", tk.END)
    calc_box.insert(tk.END, datastr)

def shiftIndexRight():
    global index, pages

    if pages == None:
        return

    if len(pages)-1 == index:
        index = 0
    else:
        index += 1

    # Clear graph from canvas
    for widget in graph_box.winfo_children():
        widget.destroy()

    displayData()

def shiftIndexLeft():
    global index, pages

    if pages == None:
        return

    if index == 0:
        index = len(pages)-1
    else:
        index -= 1

    # Clear graph from canvas
    for widget in graph_box.winfo_children():
        widget.destroy()

    displayData()

def getData():
    global sec_data, grp_data, pages, index

    path = dir_box.get()

    if not path.endswith(".RUN"):
        calc_box.delete("1.0", tk.END)
        calc_box.insert(tk.END, "Not a valid RUN file")
    else:
        data = None
        try:
            data = main.fetch(dir_box.get())
        except Exception:
            calc_box.delete("1.0", tk.END)
            calc_box.insert(tk.END, "Error reading RUN file")
            return
        sec_data = data[0]
        grp_data = data[1]
        pages = list(grp_data.keys())
        pages.extend(list(sec_data.keys()))
        index = 0
        displayData()

#___________________________________________________________ATTACH________________________________________________________#

# Attach select_file function to browse button
browse_button.config(command=select_file)

# Attach clear_text function to clear button
clear_button.config(command=clear_text)

# Attach calc_text function to calc button
calc_button.config(command= getData)

# Attach clear_canvas function to arrow buttons
left_button.config(command = (shiftIndexLeft))
right_button.config(command = shiftIndexRight)

#__________________________________________________________WEIGHTS________________________________________________________#

# Set grid weights to allow for resizing
root.columnconfigure(0, weight=1)
root.columnconfigure(1, weight=2)
root.rowconfigure(3, weight=1)

root.mainloop()


import logging
import numpy as num
from scipy import signal

from matplotlib import cm, pyplot as plt

from pyrocko.guts import Tuple, Float, Int, StringChoice, Bool
from pyrocko.plot import mpl_margins, mpl_graph_color, mpl_init

from grond.plot.config import PlotConfig
from grond.plot.collection import PlotItem

logger = logging.getLogger('grond.problem.plot')

guts_prefix = 'grond'

def fixlim(lo, hi):
    if lo == hi:
        return lo - 1.0, hi + 1.0
    else:
        return lo, hi

class SequencePlot(PlotConfig):
    '''
    Draws all parameter values evaluated during the optimisation

    The sequence of all the parameter values is either a function of the
    optimisation in progress or of the misfit from high to low. This plot can
    be used to check on convergence or see if model parameters push the given
    bounds. The color always shows the relative misfit. Relatively high misfits
    are in cold blue colors and relatively low misfits in red. The last panel
    gives the corresponding misfit values.
    '''

    name = 'sequence'
    size_cm = Tuple.T(2, Float.T(), default=(14., 6.))
    misfit_cutoff = Float.T(optional=True)
    ibootstrap = Int.T(optional=True)
    sort_by = StringChoice.T(
        choices=['iteration', 'misfit'],
        default='iteration')
    subplot_layout = Tuple.T(2, Int.T(), default=(1, 1))
    marker_size = Float.T(default=1.5)
    show_reference = Bool.T(default=True)

    def make(self, environ):
        cm = environ.get_plot_collection_manager()
        history = environ.get_history()
        optimiser = environ.get_optimiser()

        mpl_init(fontsize=self.font_size)
        cm.create_group_mpl(
            self,
            self.draw_figures(history, optimiser),
            title=u'Sequence Plots',
            section='optimiser',
            description=u'''
Sequence plots for all parameters of the optimisation.

The sequence of all the parameter values is either a function of the
optimisation in progress or of the misfit from high to low. This plot can be
used to check on convergence or to see if model parameters push the given
bounds.

The color always shows the relative misfit. Relatively high misfits are in
cold blue colors and relatively low misfits in red. The last panel gives the
corresponding misfit values.
''',
            feather_icon='fast-forward')

    def draw_figures(self, history, optimiser):
        misfit_cutoff = self.misfit_cutoff
        sort_by = self.sort_by

        problem = history.problem
        models = history.models

        npar = problem.nparameters
        ndep = problem.ndependants
        fontsize = self.font_size
        nfx, nfy = self.subplot_layout

        imodels = num.arange(history.nmodels)
        bounds = problem.get_combined_bounds()

        xref = problem.get_reference_model()

        gms = history.get_primary_chain_misfits()
        gms_softclip = num.where(gms > 1.0, 0.2 * num.log10(gms) + 1.0, gms)

        isort = num.argsort(gms)[::-1]

        if sort_by == 'iteration':
            imodels = imodels[isort]
        elif sort_by == 'misfit':
            imodels = num.arange(imodels.size)
        else:
            assert False

        gms = gms[isort]
        gms_softclip = gms_softclip[isort]
        models = models[isort, :]

        iorder = num.empty_like(isort)
        iorder = num.arange(iorder.size)

        if misfit_cutoff is None:
            ibest = num.ones(gms.size, dtype=num.bool)
        else:
            ibest = gms < misfit_cutoff

        def config_axes(axes, nfx, nfy, impl, iplot, nplots):
            if (impl - 1) % nfx != nfx - 1:
                axes.get_yaxis().tick_left()

            if (impl - 1) >= (nfx * (nfy - 1)) or iplot >= nplots - nfx:
                axes.set_xlabel('Iteration')
                if not (impl - 1) // nfx == 0:
                    axes.get_xaxis().tick_bottom()
            elif (impl - 1) // nfx == 0:
                axes.get_xaxis().tick_top()
                axes.set_xticklabels([])
            else:
                axes.get_xaxis().set_visible(False)

        # nfz = (npar + ndep + 1 - 1) / (nfx*nfy) + 1
        cmap = cm.YlOrRd
        cmap = cm.jet
        msize = self.marker_size
        axes = None
        fig = None
        item_fig = None
        nfigs = 0
        alpha = 0.5
        for ipar in range(npar):
            impl = ipar % (nfx * nfy) + 1

            if impl == 1:
                if item_fig:
                    yield item_fig
                    nfigs += 1

                fig = plt.figure(figsize=self.size_inch)
                labelpos = mpl_margins(
                    fig, nw=nfx, nh=nfy,
                    left=7.,
                    right=2.,
                    top=1.,
                    bottom=5.,
                    wspace=7., hspace=2., units=fontsize)

                item = PlotItem(name='fig_%i' % (nfigs+1))
                item.attributes['parameters'] = []
                item_fig = (item, fig)

            par = problem.parameters[ipar]

            item_fig[0].attributes['parameters'].append(par.name)

            axes = fig.add_subplot(nfy, nfx, impl)
            labelpos(axes, 2.5, 2.0)

            axes.set_ylabel(par.get_label())
            axes.get_yaxis().set_major_locator(plt.MaxNLocator(4))

            config_axes(axes, nfx, nfy, impl, ipar, npar + ndep + 1)

            axes.set_ylim(*fixlim(*par.scaled(bounds[ipar])))
            axes.set_xlim(0, history.nmodels)

            axes.scatter(
                imodels[ibest], par.scaled(models[ibest, ipar]), s=msize,
                c=iorder[ibest], edgecolors='none', cmap=cmap, alpha=alpha,
                rasterized=True)

            if self.show_reference:
                axes.axhline(par.scaled(xref[ipar]), color='black', alpha=0.3)

        for idep in range(ndep):
            # ifz, ify, ifx = num.unravel_index(ipar, (nfz, nfy, nfx))
            impl = (npar + idep) % (nfx * nfy) + 1

            if impl == 1:
                if item_fig:
                    yield item_fig
                    nfigs += 1

                fig = plt.figure(figsize=self.size_inch)
                labelpos = mpl_margins(
                    fig, nw=nfx, nh=nfy,
                    left=7.,
                    right=2.,
                    top=1.,
                    bottom=5.,
                    wspace=7., hspace=2., units=fontsize)

                item = PlotItem(name='fig_%i' % (nfigs+1))
                item.attributes['parameters'] = []

                item_fig = (item, fig)

            par = problem.dependants[idep]
            item_fig[0].attributes['parameters'].append(par.name)

            axes = fig.add_subplot(nfy, nfx, impl)
            labelpos(axes, 2.5, 2.0)

            axes.set_ylabel(par.get_label())
            axes.get_yaxis().set_major_locator(plt.MaxNLocator(4))

            config_axes(axes, nfx, nfy, impl, npar + idep, npar + ndep + 1)

            axes.set_ylim(*fixlim(*par.scaled(bounds[npar + idep])))
            axes.set_xlim(0, history.nmodels)

            ys = problem.make_dependant(models[ibest, :], par.name)
            axes.scatter(
                imodels[ibest], par.scaled(ys), s=msize, c=iorder[ibest],
                edgecolors='none', cmap=cmap, alpha=alpha, rasterized=True)

            if self.show_reference:
                y = problem.make_dependant(xref, par.name)
                axes.axhline(par.scaled(y), color='black', alpha=0.3)

        impl = (npar + ndep) % (nfx * nfy) + 1
        if impl == 1:
            if item_fig:
                yield item_fig
                nfigs += 1

            fig = plt.figure(figsize=self.size_inch)
            labelpos = mpl_margins(
                fig, nw=nfx, nh=nfy,
                left=7.,
                right=2.,
                top=1.,
                bottom=5.,
                wspace=7., hspace=2., units=fontsize)

            item = PlotItem(name='fig_%i' % (nfigs+1))
            item.attributes['parameters'] = []

            item_fig = (item, fig)

        axes = fig.add_subplot(nfy, nfx, impl)
        labelpos(axes, 2.5, 2.0)

        config_axes(axes, nfx, nfy, impl, npar + ndep, npar + ndep + 1)

        axes.set_ylim(0., 1.5)
        axes.set_yticks([0., 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4])
        axes.set_yticklabels(
            ['0.0', '0.2', '0.4', '0.6', '0.8', '1', '10', '100'])

        axes.scatter(
            imodels[ibest], gms_softclip[ibest], c=iorder[ibest],
            s=msize, edgecolors='none', cmap=cmap, alpha=alpha)

        axes.axhspan(1.0, 1.5, color=(0.8, 0.8, 0.8), alpha=0.2)
        axes.axhline(1.0, color=(0.5, 0.5, 0.5), zorder=2)

        axes.set_xlim(0, history.nmodels)
        axes.set_xlabel('Iteration')

        axes.set_ylabel('Misfit')

        yield item_fig
        nfigs += 1

class ContributionsPlot(PlotConfig):
    ''' Relative contribution of single targets to the global misfit
    '''

    name = 'contributions'
    size_cm = Tuple.T(2, Float.T(), default=(21., 14.9))

    def make(self, environ):
        cm = environ.get_plot_collection_manager()
        history = environ.get_history()
        optimiser = environ.get_optimiser()
        dataset = environ.get_dataset()

        environ.setup_modelling()

        mpl_init(fontsize=self.font_size)
        cm.create_group_mpl(
            self,
            self.draw_figures(dataset, history, optimiser),
            title=u'Target Contributions',
            section='solution',
            feather_icon='thermometer',
            description=u'''
Contributions of the targets to the total misfit.

The relative contribution that each single target has in the global misfit
result is plotted relative and unscales as a function of global misfit
(descending).

The target contribution is shown in color-filled curves with the bottom curve
on the bottom and the best-fit target on top. This plot can be used to analyse
the balance of targets in the optimisations. For ideal configurations, the
target contributions are of similar size. If the contribution of a single
target is much larger than those of all others, the weighting should be
modified.
''')

    def draw_figures(self, dataset, history, optimiser):

        fontsize = self.font_size

        fig = plt.figure(figsize=self.size_inch)
        labelpos = mpl_margins(fig, nw=2, nh=2, w=7., h=5., wspace=2.,
                               hspace=5., units=fontsize)

        problem = history.problem
        if not problem:
            logger.warn('Problem not set.')
            return []

        models = history.models

        if models.size == 0:
            logger.warn('Empty models vector.')
            return []

        for target in problem.targets:
            target.set_dataset(dataset)

        imodels = num.arange(history.nmodels)

        gms = history.get_sorted_primary_misfits()[::-1]
        isort = history.get_sorted_misfits_idx(chain=0)[::-1]

        gms **= problem.norm_exponent
        gms_softclip = num.where(gms > 1.0, 0.1 * num.log10(gms) + 1.0, gms)

        gcms = problem.combine_misfits(
            history.misfits,
            extra_correlated_weights=optimiser.get_correlated_weights(problem),
            get_contributions=True)

        gcms = gcms[isort, :]
        nmisfits = gcms.shape[1]  # noqa

        ncontributions = sum([1 if t.plot_misfits_cumulative else t.nmisfits
                              for t in problem.targets])
        cum_gcms = num.zeros((history.nmodels, ncontributions))

        # Squash matrix and sum large targets.nmisifts, eg SatelliteTarget
        plot_target_labels = []
        idx = 0
        idx_cum = 0
        for itarget, target in enumerate(problem.targets):
            target_gcms = gcms[:, idx:idx+target.nmisfits]
            if target.plot_misfits_cumulative:
                cum_gcms[:, idx_cum] = target_gcms.sum(axis=1)
                plot_target_labels.append(target.string_id())
                idx_cum += 1
            else:
                cum_gcms[:, idx_cum:idx_cum+target.nmisfits] = target_gcms
                plot_target_labels.extend(target.misfits_string_ids())
                idx_cum += target.nmisfits
            idx += target.nmisfits

        jsort = num.argsort(cum_gcms[-1, :])[::-1]

        # ncols = 4
        # nrows = ((problem.ntargets + 1) - 1) / ncols + 1

        axes = fig.add_subplot(2, 2, 1)
        labelpos(axes, 2.5, 2.0)

        axes.set_ylabel('Relative contribution (smoothed)')
        axes.set_ylim(0.0, 1.0)

        axes2 = fig.add_subplot(2, 2, 3, sharex=axes)
        labelpos(axes2, 2.5, 2.0)

        axes2.set_xlabel(
            'Tested model, sorted descending by global misfit value')

        axes2.set_ylabel('Square of misfit')

        axes2.set_ylim(0., 1.5)
        axes2.axhspan(1.0, 1.5, color=(0.8, 0.8, 0.8))
        axes2.set_yticks(
            [0., 0.2, 0.4, 0.6, 0.8, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5])
        axes2.set_yticklabels(
            ['0.0', '0.2', '0.4', '0.6', '0.8', '1', '10', '100', '1000',
             '10000', '100000'])

        axes2.set_xlim(imodels[0], imodels[-1])

        rel_ms_sum = num.zeros(history.nmodels)
        rel_ms_smooth_sum = num.zeros(history.nmodels)
        ms_smooth_sum = num.zeros(history.nmodels)
        b = num.hanning(min(100, history.nmodels//5))
        b /= num.sum(b)
        a = [1]
        ii = 0

        for idx in jsort:
            target_label = plot_target_labels[idx]
            ms = cum_gcms[:, idx]

            ms = num.where(num.isfinite(ms), ms, 0.0)
            if num.all(ms == 0.0):
                continue

            rel_ms = ms / gms

            if b.shape[0] > 5:
                rel_ms_smooth = signal.filtfilt(b, a, rel_ms)
            else:
                rel_ms_smooth = rel_ms

            ms_smooth = rel_ms_smooth * gms_softclip

            rel_poly_y = num.concatenate(
                [rel_ms_smooth_sum[::-1], rel_ms_smooth_sum + rel_ms_smooth])
            poly_x = num.concatenate([imodels[::-1], imodels])

            add_args = {}
            if ii < 20:
                add_args['label'] = '%s (%.2g)' % (
                    target_label, num.mean(rel_ms[-1]))

            axes.fill(
                poly_x, rel_poly_y,
                alpha=0.5,
                color=mpl_graph_color(ii),
                **add_args)

            poly_y = num.concatenate(
                [ms_smooth_sum[::-1], ms_smooth_sum + ms_smooth])

            axes2.fill(poly_x, poly_y, alpha=0.5, color=mpl_graph_color(ii))

            rel_ms_sum += rel_ms

            # axes.plot(
            #    imodels, rel_ms_sum, color='black', alpha=0.1, zorder=-1)

            ms_smooth_sum += ms_smooth
            rel_ms_smooth_sum += rel_ms_smooth
            ii += 1

        axes.legend(
            title='Contributions (top twenty)',
            bbox_to_anchor=(1.05, 0.0, 1.0, 1.0),
            loc='upper left',
            ncol=1, borderaxespad=0., prop={'size': 9})

        axes2.plot(imodels, gms_softclip, color='black')
        axes2.axhline(1.0, color=(0.5, 0.5, 0.5))

        return [[PlotItem(name='main'), fig]]

class BootstrapPlot(PlotConfig):
    '''
    Sorted misfit (descending) of single bootstrap chains

    For each bootstrap configuration, all models are sorted according to their
    misfit value (red lines) and their global misfit value (black line). (They
    are sorted individually for each line). The best model of every bootstrap
    configuration (right end model of red lines) is marked as a cross in the
    global misfit configuration. The horizontal black lines indicate mean and
    +- standard deviation of the y-axis values of these crosses. If the
    bootstrap configurations converge to the same region in model-space, all
    crosses should be close to the right end of the plot. If this is not the
    case, some bootstrap configurations have converged to very different places
    in model-space. This would be an indicator that there might be
    inconsistencies in the observations (maybe due to faulty or noisy or
    misoriented data). Also the shape of the curve in general can give
    information. A well-behaved optimisation run has approximately linear
    functions in this plot. Only at the end they should have a higher downward
    gradient. This would be the place where the objective functions of the
    bootstrap start to disagree.
    '''

    name = 'bootstrap'
    size_cm = Tuple.T(2, Float.T(), default=(21., 14.9))
    show_ticks = Bool.T(default=False)

    def make(self, environ):
        cm = environ.get_plot_collection_manager()
        history = environ.get_history()
        optimiser = environ.get_optimiser()
        mpl_init(fontsize=self.font_size)
        cm.create_group_mpl(
            self,
            self.draw_figures(history, optimiser),
            title=u'Bootstrap Misfit',
            section='optimiser',
            feather_icon='trending-down',
            description=u'''
Sorted misfit (descending) of single bootstrap chains.

For each bootstrap configuration, all models are sorted according to their
misfit value (red lines) and their global misfit value (black line). (They are
sorted individually for each line). The best model of every bootstrap
configuration (right end model of red lines) is marked as a cross in the global
misfit configuration. The horizontal black lines indicate mean and +- standard
deviation of the y-axis values of these crosses.

If the bootstrap configurations converge to the same region in model-space, all
crosses should be close to the right end of the plot. If this is not the case,
some bootstrap configurations have converged to very different places in
model-space. This would indicate that there might be inconsistencies in the
observations (maybe due to faulty or noisy or misoriented data). Also the shape
of the curve in general can give information. A well-behaved optimisation run
has approximately linear functions in this plot. Only at the end they should
have a higher downward gradient. This would be the place where the objective
functions of the bootstrap start to disagree.
''')

    def draw_figures(self, history, optimiser):

        fig = plt.figure()

        imodels = num.arange(history.nmodels)
        gms = history.bootstrap_misfits[:, 0]

        gms_softclip = num.where(gms > 1.0,
                                 0.1 * num.log10(gms) + 1.0,
                                 gms)
        axes = fig.add_subplot(1, 1, 1)

        ibests = []
        for ibootstrap in range(history.nchains):
            # if ibootstrap ==0:
            #    global, no-bootstrapping misfits, chain
            #    gms = history.bootstrap_misfits[:, ibootstrap]
            #    gms_softclip = num.where(gms > 1.0,
            #                            0.1 * num.log10(gms) + 1.0,
            #                            gms)

            bms = history.bootstrap_misfits[:, ibootstrap]
            isort_bms = num.argsort(bms)[::-1]
            ibests.append(isort_bms[-1])

            bms_softclip = num.where(
                bms > 1.0, 0.1 * num.log10(bms) + 1.0, bms)
            axes.plot(imodels, bms_softclip[isort_bms], color='red', alpha=0.2)

        isort = num.argsort(gms)[::-1]
        iorder = num.empty(isort.size)
        iorder[isort] = imodels

        axes.plot(iorder[ibests], gms_softclip[ibests], 'x', color='black')

        m = num.median(gms_softclip[ibests])
        s = num.std(gms_softclip[ibests])
        axes.axhline(m + s, color='black', alpha=0.5)
        axes.axhline(m, color='black')
        axes.axhline(m - s, color='black', alpha=0.5)

        axes.plot(imodels, gms_softclip[isort], color='black')

        axes.set_xlim(imodels[0], imodels[-1])
        axes.set_xlabel(
            'Tested model, sorted descending by global misfit value')

        return [(PlotItem(name='main'), fig)]

def get_plot_classes():
    return [SequencePlot, ContributionsPlot, BootstrapPlot]


from django.urls import path
from . import views

# this like app.use() in express
urlpatterns = [
    path('', views.Home.as_view(), name="home"),
    path('folios/', views.FolioList.as_view(), name="folios"),
    path('folios/new/', views.FolioCreate.as_view(), name="new_folio"),
    path('folios/<int:pk>/', views.FolioDetail.as_view(), name="folio_detail"),
    path('folios/<int:pk>/update', views.FolioUpdate.as_view(), name="folio_update"),
    path('folios/<int:pk>/delete', views.FolioDelete.as_view(), name="folio_delete"),
    path('folios/<int:pk>/new/', views.QuartetCreate.as_view(), name="new_quartet"),
    path('folios/<int:folio_pk>/<int:pk>', views.QuartetDetail.as_view(), name="quartet_detail"),
    path('folios/<int:folio_pk>/<int:pk>/update', views.QuartetUpdate.as_view(), name="quartet_update"),
    path('folios/<int:folio_pk>/<int:pk>/delete', views.QuartetDelete.as_view(), name="quartet_delete"),
    path('folios/<int:folio_pk>/<int:pk>/new_text_entry/', views.TextEntryCreate.as_view(), name="new_text_entry"),
    path('folios/<int:folio_pk>/<int:pk>/new_image_entry/', views.ImageEntryCreate.as_view(), name="new_image_entry"),
    path('folios/<int:folio_pk>/<int:pk>/new_embed_entry/', views.EmbedEntryCreate.as_view(), name="new_embed_entry"),
    path('folios/<int:folio_pk>/<int:pk>/new_video_entry/', views.VideoEntryCreate.as_view(), name="new_video_entry"),
    path('folios/<int:folio_pk>/<int:quartet_pk>/<int:pk>/delete/', views.EntryDelete.as_view(), name="entry_delete"),
    path('accounts/signup/', views.Signup.as_view(), name="signup")
]


import time

from stationexec.sequencer.operation import Operation, require_tools, OperationState

version = "0.1"

@require_tools("chassis_1")
class PowerTest(Operation):
    def operation_action(self):
        self.chassis_1.set_power(200)
        self.chassis_1.run_ttv(1)
        time.sleep(5)
        self.chassis_1.run_ttv(0)


from aiogram import types
from telegram_bot_pagination import InlineKeyboardPaginator

def get_pagination_buttons(
    page_count: int, current_page: int, data_pattern: str, cls=InlineKeyboardPaginator
):
    paginator = cls(page_count, current_page=current_page, data_pattern=data_pattern)
    pagination_buttons = [
        types.InlineKeyboardButton(text=button["text"], callback_data=button["callback_data"])
        for button in paginator.keyboard
    ]

    return pagination_buttons


#!/usr/bin/env python3
import math

import numpy
import roslib
import rospy
from geometry_msgs.msg import (
    Point,
    Pose,
    PoseWithCovariance,
    Quaternion,
    Twist,
    TwistWithCovariance,
    Vector3,
    WrenchStamped,
)
from interactive_markers.interactive_marker_server import InteractiveMarkerServer
from mil_msgs.msg import PoseTwist, PoseTwistStamped
from mil_msgs.orientation_helpers import xyz_array, xyzw_array
from nav_msgs.msg import Odometry
from std_msgs.msg import ColorRGBA, Header
from tf import transformations
from visualization_msgs.msg import (
    InteractiveMarker,
    InteractiveMarkerControl,
    InteractiveMarkerFeedback,
    Marker,
)

def makeBoxControl() -> InteractiveMarkerControl:
    return InteractiveMarkerControl(
        always_visible=True,
        markers=[
            Marker(
                type=Marker.CUBE,
                scale=Vector3(0.45, 0.45, 0.45),
                color=ColorRGBA(0.5, 0.5, 0.5, 1),
            ),
        ],
    )

def make6DofMarker(
    name: str,
    description: str,
    allow_rotation: bool = False,
) -> InteractiveMarker:
    int_marker = InteractiveMarker()
    int_marker.name = name
    int_marker.description = description
    int_marker.header.frame_id = "map"
    int_marker.scale = 1

    # insert a box
    int_marker.controls.append(makeBoxControl())

    for direction in "xyz":
        control = InteractiveMarkerControl()
        control.orientation.w = 1
        control.orientation.x = direction == "x"
        control.orientation.y = direction == "y"
        control.orientation.z = direction == "z"
        control.name = "move_" + direction
        control.interaction_mode = InteractiveMarkerControl.MOVE_AXIS
        int_marker.controls.append(control)

        if allow_rotation:
            control = InteractiveMarkerControl()
            control.orientation.w = 1
            control.orientation.x = direction == "x"
            control.orientation.y = direction == "y"
            control.orientation.z = direction == "z"
            control.name = "rotate_" + direction
            control.interaction_mode = InteractiveMarkerControl.ROTATE_AXIS
            int_marker.controls.append(control)

    return int_marker

def makeMovingMarker(name: str, description: str) -> InteractiveMarker:
    int_marker = InteractiveMarker()
    int_marker.name = name
    int_marker.description = description
    int_marker.header.frame_id = "map"
    int_marker.scale = 1

    # insert a box
    int_marker.controls.append(makeBoxControl())

    return int_marker

if __name__ == "__main__":
    roslib.load_manifest("interactive_markers")
    roslib.load_manifest("rise_6dof")

    rospy.init_node("test_rviz")
    server = InteractiveMarkerServer("test_rviz")

    wrench = [(0, 0, 0), (0, 0, 0)]

    def set_wrench(new_wrench):
        assert new_wrench.header.frame_id == "/base_link"
        new_wrench = new_wrench.wrench
        wrench[:] = [
            (new_wrench.force.x, new_wrench.force.y, new_wrench.force.z),
            (new_wrench.torque.x, new_wrench.torque.y, new_wrench.torque.z),
        ]

    wrench_sub = rospy.Subscriber("/output", WrenchStamped, set_wrench)

    desired_pub = rospy.Publisher("/desired", PoseTwistStamped)
    desired_posetwist = PoseTwistStamped(
        header=Header(frame_id="map", stamp=rospy.Time.now()),
        posetwist=PoseTwist(
            pose=Pose(position=Point(0, 0, 0), orientation=Quaternion(0, 0, 0, 1)),
            twist=Twist(linear=Vector3(0, 0, 0), angular=Vector3(0, 0, 0)),
        ),
    )

    def updateDesired(feedback=None):
        global desired_posetwist
        if (
            feedback is not None
            and feedback.event_type == InteractiveMarkerFeedback.POSE_UPDATE
        ):
            # compute twist from this and last pose update
            if desired_posetwist is not None and rospy.Duration.from_sec(
                0,
            ) < rospy.Time.now() - desired_posetwist.header.stamp < rospy.Duration.from_sec(
                0.05,
            ):
                dt = (rospy.Time.now() - desired_posetwist.header.stamp).to_sec()
                vel = (
                    xyz_array(feedback.pose.position)
                    - xyz_array(desired_posetwist.posetwist.pose.position)
                ) / dt

                def quat_to_scaledaxis(q):
                    q = q / numpy.linalg.norm(q)
                    if q[3] < 0:
                        q *= -1
                    return 2 / numpy.sinc(q[3] / math.pi) * numpy.array(q[:3])

                angvel = (
                    quat_to_scaledaxis(
                        transformations.quaternion_multiply(
                            xyzw_array(feedback.pose.orientation),
                            transformations.quaternion_conjugate(
                                xyzw_array(
                                    desired_posetwist.posetwist.pose.orientation,
                                ),
                            ),
                        ),
                    )
                    / dt
                )
                twist = Twist(linear=Vector3(*vel), angular=Vector3(*angvel))
            else:
                twist = Twist(linear=Vector3(0, 0, 0), angular=Vector3(0, 0, 0))
            desired_posetwist = PoseTwistStamped(
                header=Header(frame_id="map", stamp=rospy.Time.now()),
                posetwist=PoseTwist(
                    pose=feedback.pose,
                    twist=twist,
                ),
            )
        if rospy.Time.now() - desired_posetwist.header.stamp > rospy.Duration.from_sec(
            0.05,
        ):
            desired_posetwist.posetwist.twist = Twist(
                linear=Vector3(0, 0, 0),
                angular=Vector3(0, 0, 0),
            )
        if desired_posetwist is not None:
            desired_pub.publish(desired_posetwist)

    server.insert(make6DofMarker("desired", "\nDesired", True), updateDesired)
    rospy.Timer(rospy.Duration(0.01), lambda msg: updateDesired(None))

    current_pub = rospy.Publisher("/current", Odometry)
    server.insert(makeMovingMarker("current", "Current\n"))

    current_pos = numpy.zeros(3)
    current_vel = numpy.zeros(3)
    current_orientation = numpy.array([0, 0, 0, 1])
    current_angvel = numpy.zeros(3)
    vel_m = 10
    angvel_I = 10
    dt = 0.01

    def updateCurrent(msg):
        global current_pos, current_vel, current_orientation, current_angvel

        world_from_body = transformations.quaternion_matrix(current_orientation)[:3, :3]

        # print wrench
        wrench_world = world_from_body.dot(wrench[0]), world_from_body.dot(wrench[1])

        current_vel += dt * (wrench_world[0] - 10 * current_vel) / vel_m
        current_angvel += dt * (wrench_world[1] - 5 * current_angvel) / angvel_I

        current_pos += dt * current_vel

        def quat_exp(quat):
            x, y, z = quat
            return transformations.quaternion_about_axis(
                numpy.linalg.norm((x, y, z)),
                (x, y, z),
            )

        current_orientation = transformations.quaternion_multiply(
            quat_exp(current_angvel * dt),
            current_orientation,
        )

        pose = Pose(
            Point(x=current_pos[0], y=current_pos[1], z=current_pos[2]),
            Quaternion(
                x=current_orientation[0],
                y=current_orientation[1],
                z=current_orientation[2],
                w=current_orientation[3],
            ),
        )

        server.setPose("current", pose)
        server.applyChanges()

        current_pub.publish(
            Odometry(
                header=Header(frame_id="map"),
                pose=PoseWithCovariance(pose=pose),
                twist=TwistWithCovariance(
                    twist=Twist(
                        linear=Vector3(*current_vel),
                        angular=Vector3(*current_angvel),
                    ),
                ),
            ),
        )

    rospy.Timer(rospy.Duration(dt), updateCurrent)

    server.applyChanges()
    rospy.spin()


# n=int(input("Enter a number: "))
# count=0
# for i in range(2,n//2):
#     if n%i==0:
#         count=1
#         break
# print("Prime number") if count==0 else print("non prime number")

# Find prime numbers in a range
l=[]
for i in range(50,101):
    for j in range(2,i//2):
        if (i%j)==0:
            break
    else:
        l.append(i)
print(l)


import math

import matplotlib.pylab as plt
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sympy import symbols, solve

print("Question 2")
my_id = "09170218"  # defining U
U = [int(u) for u in my_id]
X = [x for x in range(1, 9)]

plt.plot(X, U)
plt.xlabel("X")
plt.ylabel("U(x)")
plt.title("U_X graph")
plt.show()

"""
The mechanical energy (E = U + K) to maximize amplitude:
   we set E=5 (U(5) = 0) ==> Xl = 4.7 and Xr = 6
   ATTENTION: (4.7, 2) this point is approximated by my eyes.
   and we get three points to fit a quadratic function to them (4.7, 2), (5, 0) and (6, 2)
"""

X = np.matrix([4.7, 5, 6]).reshape((3, 1))
U = np.matrix([2, 0, 2]).reshape((3, 1))
a = PolynomialFeatures(degree=2)
a.fit(X, U)
model = make_pipeline(a, LinearRegression())
model.fit(X, U)

alpha = model.steps[1][1].coef_[0][2]
beta = model.steps[1][1].coef_[0][1]
phi = model.steps[1][1].intercept_[0]

print("alpha: ", alpha, " beta: ", beta, " landa: ", phi)

def U(x):
    return alpha * x ** 2 + beta * x + phi

"""
At equilibrium point we have dU(x)/dx = 0
"""

equilibrium_point = symbols('equilibrium_point')
equation = 2 * alpha * equilibrium_point + beta  # dU(x)/dx = 0
find_equilibrium_point = solve([equation])
print("find_equilibrium_point: ", find_equilibrium_point.get(equilibrium_point))

"""
To estimate the frequency of the harmonic oscillations  we do the following:
    we have to convert the quadratic function into this form
    U = 1/2 * k * (x-x0)^2 opening up this equation we get
    alpha = 1/2 * k * x**2
    beta = -1/2 * k * x0 * x
    phi = 1/2 * k * x0**2
"""
k = 2 * alpha
mass = 1
print("k: ", k)
frequency_of_the_harmonic_oscillations = 1 / (2 * math.pi) * math.sqrt(k / mass)
print("frequency_of_the_harmonic_oscillations: ", frequency_of_the_harmonic_oscillations)

print('\n\n\n')
###########  Question 3 ###########################
print("Question 3")
h = 10
M = 10
"""
celli = [x_coordinate, y_coordinate, mass_of_the_cell]
N = [0,1,2,3,4,5,6,7,8,9]
mass of i are all equal if i= N - {4,5} = M/16
mass of i are all equal if i= {4,5} = M/4
"""
cell0 = [h / 8, 7 * h / 8, M / 16]
cell1 = [3 * h / 8, 7 * h / 8, M / 16]
cell2 = [5 * h / 8, 7 * h / 8, M / 16]
cell3 = [7 * h / 8, 7 * h / 8, M / 16]
cell4 = [h / 4, h / 2, M / 4]
cell5 = [3 * h / 4, h / 2, M / 4]
cell6 = [h / 8, h / 8, M / 16]
cell7 = [3 * h / 8, h / 8, M / 16]
cell8 = [5 * h / 8, h / 8, M / 16]
cell9 = [7 * h / 8, h / 8, M / 16]

"""
cell8 is the one that must be removed owing to my id ==>
"""
coordinates_of_all_cells = [cell0, cell1, cell2, cell3, cell4, cell5, cell6, cell7, cell9]

x_of_center_of_mass = 0
center_of_mass = 0
for cell in coordinates_of_all_cells:
    x_of_center_of_mass += cell[0] * cell[2]
    center_of_mass += cell[2]
x_of_center_of_mass = x_of_center_of_mass / center_of_mass

y_of_center_of_mass = 0
center_of_mass = 0
for cell in coordinates_of_all_cells:
    y_of_center_of_mass += cell[1] * cell[2]
    center_of_mass += cell[2]
y_of_center_of_mass = y_of_center_of_mass / center_of_mass

"""Now I want to show all the points (except cell8)"""

all_the_x = [cell[0] for cell in coordinates_of_all_cells]
all_the_y = [cell[1] for cell in coordinates_of_all_cells]
center_of_mass_coordinates = [x_of_center_of_mass, y_of_center_of_mass]

plt.scatter(all_the_x, all_the_y)
plt.xlabel("X")
plt.ylabel("Y")
plt.title("X_Y graph")

plt.scatter(center_of_mass_coordinates[0], center_of_mass_coordinates[1], edgecolors='r')
plt.show()

print('\n\n\n')
################# Question 4 ################################
print("Question 4")

"""mi = [ x_coordinate_of_mi, y_coordinate_of_mi, mass_of_mi ] """
m1 = [0, 9, 1]
m2 = [1, 7, 2]
m3 = [0, 2, 2]
m4 = [1, 8, 1]

all_the_bodies = [m1, m2, m3, m4]
all_the_bodies_x = [body[0] for body in all_the_bodies]
all_the_bodies_y = [body[1] for body in all_the_bodies]

# part 1: will show all the points later on

# part 2:

center_of_mass_x = 0
center_of_mass = 0
for body in all_the_bodies:
    center_of_mass_x += body[0] * body[2]
    center_of_mass += body[2]
center_of_mass_x = center_of_mass_x / center_of_mass

center_of_mass_y = 0
center_of_mass = 0
for body in all_the_bodies:
    center_of_mass_y += body[1] * body[2]
    center_of_mass += body[2]
center_of_mass_y = center_of_mass_y / center_of_mass

center_of_mass_coordinates = [center_of_mass_x,
                              center_of_mass_y]  # center of mass of the original system

plt.scatter(all_the_bodies_x, all_the_bodies_y)
plt.xlabel("X")
plt.ylabel("Y")
plt.title("X_Y graph")
plt.scatter(center_of_mass_coordinates[0], center_of_mass_coordinates[1], edgecolors='r')
plt.show()
print("center_of_mass_coordinates: ", center_of_mass_coordinates)

# part 3

def distance_between_2_points(m1, m2):
    return math.sqrt((m1[0] - m2[1]) ** 2 + (m1[2] - m1[2]) ** 2)

L1 = distance_between_2_points(m1, m2)
L2 = distance_between_2_points(m2, m3)
L3 = distance_between_2_points(m3, m4)

print('L1: ', L1)
print('L2: ', L2)
print('L3: ', L3)

"""
We have obtained that theta is angle between v1 and x-axis and is 63.43 so
    x and y components of v1 are obtained as follows:
"""
theta = 63.43
v1 = L1
v1x = abs(math.cos(theta) * v1)
v1y = abs(math.sin(theta) * v1)
v1 = [-v1x, v1y]  # negative sign is for the fact that we are on the left side of X-axis
new_position_of_m1_after_1_second = [m1[0] + v1[0], m1[1] + v1[1]]
print("new_position_of_m1_after_1_second: ", new_position_of_m1_after_1_second)

"""
We have obtained that alpha is angle between v2 and x-axis and is 80.53 so
    x and y components of v1 are obtained as follows:
"""
alpha = 80.53
v2 = L3
v2x = abs(math.cos(alpha) * v2)
v2y = abs(math.sin(alpha) * v2)
v2 = [v2x, v2y]  # positive sign is for the fact that we are on the right side of X-axis
new_position_of_m4_after_1_second = [m4[0] + v2[0], m4[1] + v2[1]]
print("new_position_of_m4_after_1_second: ", new_position_of_m4_after_1_second)

L2_center_of_mass = 0  # center of mass of the rod L2 1 second after the explosions
"""
There is no external force acting on our system ==> explosion does not change the center of mass
    ==> center_of_mass_coordinates do not change.
The logic is the following:
    we use both new_position_of_m1_after_1_second and new_position_of_m4_after_1_second to calculate
    L2_center_of_mass.
"""
total_mass = m1[2] + m2[2] + m3[2] + m4[2]

L2_center_of_mass_x = symbols('L2_center_of_mass_x')
equation_for_x = ((L2_center_of_mass_x * (m2[2] + m3[2]) + new_position_of_m1_after_1_second[0] *
                   m1[2] + new_position_of_m4_after_1_second[0] * m4[2]) / total_mass) - \
                 center_of_mass_coordinates[0]

L2_center_of_mass_y = symbols('L2_center_of_mass_y')
equation_for_y = ((L2_center_of_mass_y * (m2[2] + m3[2]) + new_position_of_m1_after_1_second[1] *
                   m1[2] + new_position_of_m4_after_1_second[1] * m4[2]) / total_mass) - \
                 center_of_mass_coordinates[1]

L2_center_of_mass_x = solve([equation_for_x]).get(L2_center_of_mass_x)
L2_center_of_mass_y = solve([equation_for_y]).get(L2_center_of_mass_y)

print("L2_center_of_mass_x after explosion: ", L2_center_of_mass_x)
print("L2_center_of_mass_y after explosion: ", L2_center_of_mass_y)

plt.scatter([L2_center_of_mass_x], [L2_center_of_mass_y], edgecolors='r')
plt.xlabel("X")
plt.ylabel("Y")
plt.title("X_Y Graph After Explosion")
plt.show()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import pigpio
import time

def main():
    # 使用する出力ピン(GPIO)
    pwm_pin = 18

    freq=50

    # GPIOピンの定義
    pi = pigpio.pi()
    pi.set_mode(pwm_pin, pigpio.OUTPUT)
    pi.hardware_PWM(pwm_pin, freq, 500000)

    for i in range(100):
        pi.set_PWM_dutycycle(pwm_pin, 0)
        time.sleep(0.5)
        pi.set_PWM_frequency(pwm_pin, freq)
        print("freq set/get =", freq, "/", pi.get_PWM_frequency(pwm_pin))
        pi.set_PWM_dutycycle(pwm_pin, 128)
        time.sleep(2)
        freq=freq+50

    pi.set_PWM_dutycycle(pwm_pin, 0)
    pi.stop()

if __name__ == "__main__":
    main()


class Node:
    def __init__(self, data):
        self.data = data
        self.next = None

def bubbleSort(head):

    end = None

    while head.next != end:

        prev = None
        current = head

        while current.next != end:

            if current.data > current.next.data and prev is None:
                temp = head
                head = head.next
                temp.next = head.next
                head.next = temp

                prev = head
                current = head.next

            elif current.data > current.next.data and prev is not None:
                prev.next = prev.next.next
                current.next = prev.next.next
                prev.next.next = current

                prev = prev.next

            else:
                prev = current
                current = current.next

        end = current

    return head

def insertNode(obj, data):
    itr = obj
    while itr.next:
        itr = itr.next
    itr.next = Node(data)

def printLL(obj):
    while obj:
        print(obj.data," --> ", end="")
        obj = obj.next
    else:
        print("Null")

if __name__ == "__main__":

    obj1 = Node(70)
    insertNode(obj1, 60)
    insertNode(obj1, 50)
    insertNode(obj1, 120)
    insertNode(obj1, 30)
    insertNode(obj1, 20)
    insertNode(obj1, 10)

    obj1 = bubbleSort(obj1)
    printLL(obj1)


import h5py
import numpy as np
import sys
import os
import json
from datetime import datetime, date
from tqdm import tqdm

def progress(count, total, status='', info=""):
    bar_len = 100
    filled_len = int(round(bar_len * count / float(total)))

    percents = round(100.0 * count / float(total), 1)
    bar = 'O' * filled_len + '.' * (bar_len - filled_len)

    _ = "{0} [{1}] {2}%".format(info, bar, percents)
    print(_, end='\r')

def json_from_file(fnm):
    r = None
    with open(fnm) as json_file:
        r = json.load(json_file)
    return r

def load_ground_stations(fnm):
    return json_from_file(fnm)

def load_omi_data(fnm, data_field):
    with h5py.File(fnm) as f:
        # Read dataset.
        dset = f[data_field]
        data = dset[:]
        # Handle fill value.
        data[data == dset.fillvalue] = np.nan
        data = np.ma.masked_where(np.isnan(data), data)
        return data

def read_station_data(station_id, date, type=["NO2"], time="13:00"):
    try:
        r = {}
        fnm = "{0}/{1}/{2}.json".format(STATION_FOLDER, station_id, date)
        station_data = json_from_file(fnm)
        for d in station_data["data"]["tabularData"]["bodyContent"]:
            if d["from date"][-5:] == time:
                for t in type:
                    r[t] = d[t]
                break
        return r
    except Exception as ex:
        return None

########################################################################################################################
OMI_DATA_FOLDER = "data/OMI"
GROUND_STATIONS = "data/Indian_station_list_OMIxy.json"
DATA_FIELD_NAME = "HDFEOS/GRIDS/ColumnAmountNO2/Data Fields/ColumnAmountNO2"
STATION_FOLDER = "data/cpcb"
PROCESSED_DATA = "data/processed_data-17-22.json"

ground_stations = load_ground_stations(GROUND_STATIONS)
ground_measure_item = ["NO2"]
ground_measure_time = "13:00"

stations_data = {}
fnm = None
omi_file_count = len(os.listdir(OMI_DATA_FOLDER))
current_count = 0

for file in os.listdir(OMI_DATA_FOLDER):
    if file.endswith(".he5"):
        fnm = os.path.join(OMI_DATA_FOLDER, file)
        omi_data = load_omi_data(fnm, DATA_FIELD_NAME)  # NO2
        # OMI-Aura_ L3-OMNO2d_ 2021m0311_ v003-2021m0421t134756.he5
        fnm_list = fnm.split("_")
        yr, mm, dd = fnm_list[2][:4], fnm_list[2][5:7], fnm_list[2][7:9]
        omi_time = fnm_list[3][-10:][:6]

        for station in ground_stations:
            if station["station_id"] not in stations_data:
                stations_data[station["station_id"]] = station.copy()

            st = stations_data[station["station_id"]]
            ground_data = read_station_data(station["station_id"],
                                            "{0}-{1}-{2}".format(dd, mm, yr),
                                            ground_measure_item,
                                            ground_measure_time)
            if "date" in st:
                st["date"].append("{0}/{1}/{2}".format(yr, mm, dd))
                st["OMI_NO2"].append(str(omi_data[station["OMI_y"]][station["OMI_x"]]))
                # st["time"].append(omi_time)
                for item in ground_measure_item:
                    if ground_data:
                        st[item].append(ground_data[item])
                    else:
                        st[item].append(None)
            else:
                st["date"] = ["{0}/{1}/{2}".format(yr, mm, dd)]
                st["OMI_NO2"] = [str(omi_data[station["OMI_y"]][station["OMI_x"]])]
                # st["time"] = [omi_time]
                for item in ground_measure_item:
                    if ground_data:
                        st[item] = [ground_data[item]]
                    else:
                        st[item] = [None]
        # stations_data[st["station_id"]] = st
        current_count += 1
        progress(current_count, omi_file_count, "", "{0}/{1}/{2}".format(mm, dd, yr))
        # if current_count > 10:
        #     break

if stations_data:
    with open(PROCESSED_DATA, "w") as json_file:
        json_file.write(json.dumps(stations_data))


#
# 正则表达式 : Regular Expression
#   作用：匹配指定规则的字符串
# 爬虫，匹配手机号，邮箱，qq号等
#

# 验证手机号码是否为11位，且第一位是1，都为数字
def fn(phone):
    if len(phone) != 11:
        return False
    if phone[0] != '1':
        return False
    if not phone.isdigit():
        return False
    return True

fn("13456788765")

# 使用正则表达式
import re

# re.match(): 匹配字符串是否以指定正则表达式开头，类似字符串中的startswith, 如果能够匹配成功则返回对象, 否则返回None
#   参数1： 正则表达式
#   参数2： 需要匹配的字符串
#   参数3： 标记
#             re.I : ignore,忽略大小写
#             re.M : 换行模式
#             re.S : 跟符号.配合使用，让.可以匹配到换行
# res = re.match('www', 'www.baidu.com')
# res = re.match('www', 'ww.baidu.www.com')
# res = re.match('www', 'WWW.baidu.com', re.I)
# print(res)

# re.search(): 匹配字符串中是否出现指定正则表达式的内容，如果有则返回第一次匹配到的内容，否则返回None
# res = re.search("www", 'www.baidu.com')
# res = re.search("www", 'baidu.www.com')
# res = re.search("www", 'baidu.wwww.www.com')
# res = re.search("www", 'baidu.WWWW.www.com', re.I)
# res = re.search("www", 'w.baidu.com')  # None
# print(res)

# re.findall(): 获取字符串中匹配正则表达式的所有内容，如果没有匹配的则返回空列表
# res = re.findall('www', 'www.baidu.com')  # ['www']
# res = re.findall('www', 'www.baidu.WWw.com', re.I)  # ['www', 'WWw']
res = re.findall('www', 'w.baidu.Ww.com')  # []
print(res)

05_正则表达式的方法.py